Merge pull request #26913 from bprashanth/petset_e2e

Automatic merge from submit-queue Improve petset e2es Mainly * scale petset to 0 and wait for volume provisioner to delete pvs so we don't leak them * bake binaries into init[0] and cp into destination, instead of wgetting at runtime * dump verbose debug output on failure https://github.com/kubernetes/test-infra/pull/115, https://github.com/kubernetes/kubernetes/issues/26824
2016-06-09 23:31:36 -07:00
parent 805a0aae71 ff23a0e4fc
commit abc06c9d73
14 changed files with 568 additions and 52 deletions
--- a/test/images/pets/redis/Dockerfile
+++ b/test/images/pets/redis/Dockerfile
@@ -0,0 +1,39 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO: get rid of bash dependency and switch to plain busybox.
+# The tar in busybox also doesn't seem to understand compression.
+FROM debian:jessie
+MAINTAINER Prashanth.B <beeps@google.com>
+
+# TODO: just use standard redis when there is one for 3.2.0.
+RUN apt-get update && apt-get install -y wget make gcc
+
+# See README.md
+RUN wget -qO /redis-3.2.0.tar.gz http://download.redis.io/releases/redis-3.2.0.tar.gz && \
+    tar -xzf /redis-3.2.0.tar.gz -C /tmp/ && rm /redis-3.2.0.tar.gz
+
+# Clean out existing deps before installation
+# see https://github.com/antirez/redis/issues/722
+RUN cd /tmp/redis-3.2.0 && make distclean && mkdir -p /redis && \
+    make install INSTALL_BIN=/redis && \
+    mv /tmp/redis-3.2.0/redis.conf /redis/redis.conf && \
+    rm -rf /tmp/redis-3.2.0
+
+ADD on-start.sh /
+# See contrib/pets/peer-finder for details
+RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder
+ADD install.sh /
+RUN chmod -c 755 /install.sh /on-start.sh /peer-finder
+Entrypoint ["/install.sh"]
--- a/test/images/pets/redis/Makefile
+++ b/test/images/pets/redis/Makefile
@@ -0,0 +1,27 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+all: push
+
+TAG = e2e
+PREFIX = gcr.io/google_containers/redis-install-3.2.0
+
+container:
+	docker build -t $(PREFIX):$(TAG) .
+
+push: container
+	gcloud docker push $(PREFIX):$(TAG)
+
+clean:
+	docker rmi $(PREFIX):$(TAG)
--- a/test/images/pets/redis/README.md
+++ b/test/images/pets/redis/README.md
@@ -0,0 +1,12 @@
+# Redis petset e2e tester
+
+The image in this directory is the init container for contrib/pets/redis but for one difference, it bakes a specific verfion of redis into the base image so we get deterministic test results without having to depend on a redis download server. Discussing the tradeoffs to either approach (download the version at runtime, or maintain an image per version) are outside the scope of this document.
+
+You can execute the image locally via:
+```
+$ docker run -it gcr.io/google_containers/redis-install-3.2.0:e2e --cmd --install-into=/opt --work-dir=/work-dir
+```
+To share the installation with other containers mount the appropriate volumes as `--install-into` and `--work-dir`, where `install-into` is the directory to install redis into, and `work-dir` is the directory to install the user/admin supplied on-{start,change} hook scripts.
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/images/pets/redis/README.md?pixel)]()
--- a/test/images/pets/redis/install.sh
+++ b/test/images/pets/redis/install.sh
@@ -0,0 +1,51 @@
+#! /bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This volume is assumed to exist and is shared with parent of the init
+# container. It contains the redis installation.
+INSTALL_VOLUME="/opt"
+
+# This volume is assumed to exist and is shared with the peer-finder
+# init container. It contains on-start/change configuration scripts.
+WORK_DIR="/work-dir"
+
+VERSION="3.2.0"
+
+for i in "$@"
+do
+case $i in
+    -i=*|--install-into=*)
+    INSTALL_VOLUME="${i#*=}"
+    shift
+    ;;
+    -w=*|--work-dir=*)
+    WORK_DIR="${i#*=}"
+    shift
+    ;;
+    *)
+    # unknown option
+    ;;
+esac
+done
+
+echo installing config scripts into "${WORK_DIR}"
+mkdir -p "${WORK_DIR}"
+cp /on-start.sh "${WORK_DIR}"/
+cp /peer-finder "${WORK_DIR}"/
+
+echo installing redis-"${VERSION}" into "${INSTALL_VOLUME}"
+mkdir -p "${INSTALL_VOLUME}"
+mv /redis "${INSTALL_VOLUME}"/redis
--- a/test/images/pets/redis/on-start.sh
+++ b/test/images/pets/redis/on-start.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CFG=/opt/redis/redis.conf
+HOSTNAME=$(hostname)
+DATADIR="/data"
+# Port on which redis listens for connections.
+PORT=6379
+
+# Ping everyone but ourself to see if there's a master. Only one pet starts at
+# a time, so if we don't see a master we can assume the position is ours.
+while read -ra LINE; do
+    if [[ "${LINE}" == *"${HOSTNAME}"* ]]; then
+        sed -i -e "s|^bind.*$|bind ${LINE}|" ${CFG}
+    elif [ "$(/opt/redis/redis-cli -h $LINE info | grep role | sed 's,\r$,,')" = "role:master" ]; then
+        # TODO: More restrictive regex?
+        sed -i -e "s|^.*slaveof.*$|slaveof ${LINE} ${PORT}|" ${CFG}
+    fi
+done
+
+# Set the data directory for append only log and snapshot files. This should
+# be a persistent volume for consistency.
+sed -i -e "s|^.*dir .*$|dir ${DATADIR}|" ${CFG}
+
+# The append only log is written for every SET operation. Without this setting,
+# redis just snapshots periodically which is only safe for a cache. This will
+# produce an appendonly.aof file in the configured data dir.
+sed -i -e "s|^appendonly .*$|appendonly yes|" ${CFG}
+
+# Every write triggers an fsync. Recommended default is "everysec", which
+# is only safe for AP applications.
+sed -i -e "s|^appendfsync .*$|appendfsync always|" ${CFG}
+
+
--- a/test/images/pets/zookeeper/Dockerfile
+++ b/test/images/pets/zookeeper/Dockerfile
@@ -0,0 +1,32 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO: get rid of bash dependency and switch to plain busybox.
+# The tar in busybox also doesn't seem to understand compression.
+FROM debian:jessie
+MAINTAINER Prashanth.B <beeps@google.com>
+
+RUN apt-get update && apt-get install -y wget netcat
+
+ADD on-start.sh /
+# See contrib/pets/peer-finder for details
+RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder
+
+# See README.md
+RUN wget -q -O /zookeeper-3.5.0-alpha.tar.gz http://apache.mirrors.pair.com/zookeeper/zookeeper-3.5.0-alpha/zookeeper-3.5.0-alpha.tar.gz && \
+    tar -xzf /zookeeper-3.5.0-alpha.tar.gz -C /tmp/ && mv /tmp/zookeeper-3.5.0-alpha /zookeeper && rm /zookeeper-3.5.0-alpha.tar.gz
+
+ADD install.sh /
+RUN chmod -c 755 /install.sh /on-start.sh /peer-finder
+Entrypoint ["/install.sh"]
--- a/test/images/pets/zookeeper/Makefile
+++ b/test/images/pets/zookeeper/Makefile
@@ -0,0 +1,27 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+all: push
+
+TAG = e2e
+PREFIX = gcr.io/google_containers/zookeeper-install-3.5.0-alpha
+
+container:
+	docker build -t $(PREFIX):$(TAG) .
+
+push: container
+	gcloud docker push $(PREFIX):$(TAG)
+
+clean:
+	docker rmi $(PREFIX):$(TAG)
--- a/test/images/pets/zookeeper/README.md
+++ b/test/images/pets/zookeeper/README.md
@@ -0,0 +1,12 @@
+# Zookeeper petset e2e tester
+
+The image in this directory is the init container for contrib/pets/zookeeper but for one difference, it bakes a specific verfion of zookeeper into the base image so we get deterministic test results without having to depend on a zookeeper download server. Discussing the tradeoffs to either approach (download the version at runtime, or maintain an image per version) are outside the scope of this document.
+
+You can execute the image locally via:
+```
+$ docker run -it gcr.io/google_containers/zookeeper-install-3.5.0-alpha:e2e --cmd --install-into=/opt --work-dir=/work-dir
+```
+To share the installation with other containers mount the appropriate volumes as `--install-into` and `--work-dir`, where `install-into` is the directory to install zookeeper into, and `work-dir` is the directory to install the user/admin supplied on-{start,change} hook scripts.
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/images/pets/zookeeper/README.md?pixel)]()
--- a/test/images/pets/zookeeper/install.sh
+++ b/test/images/pets/zookeeper/install.sh
@@ -0,0 +1,70 @@
+#! /bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This volume is assumed to exist and is shared with parent of the init
+# container. It contains the zookeeper installation.
+INSTALL_VOLUME="/opt"
+
+# This volume is assumed to exist and is shared with the peer-finder
+# init container. It contains on-start/change configuration scripts.
+WORKDIR_VOLUME="/work-dir"
+
+# As of April-2016 is 3.4.8 is the latest stable, but versions 3.5.0 onward
+# allow dynamic reconfiguration.
+VERSION="3.5.0-alpha"
+
+for i in "$@"
+do
+case $i in
+    -i=*|--install-into=*)
+    INSTALL_VOLUME="${i#*=}"
+    shift
+    ;;
+    -w=*|--work-dir=*)
+    WORKDIR_VOLUME="${i#*=}"
+    shift
+    ;;
+    *)
+    # unknown option
+    ;;
+esac
+done
+
+echo installing config scripts into "${WORKDIR_VOLUME}"
+mkdir -p "${WORKDIR_VOLUME}"
+cp /on-start.sh "${WORKDIR_VOLUME}"/
+cp /peer-finder "${WORKDIR_VOLUME}"/
+
+echo installing zookeeper-"${VERSION}" into "${INSTALL_VOLUME}"
+mkdir -p "${INSTALL_VOLUME}"
+mv /zookeeper "${INSTALL_VOLUME}"/zookeeper
+cp "${INSTALL_VOLUME}"/zookeeper/conf/zoo_sample.cfg "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg
+
+# TODO: Should dynamic config be tied to the version?
+IFS="." read -ra RELEASE <<< "${VERSION}"
+if [ $(expr "${RELEASE[1]}") -gt 4 ]; then
+    echo zookeeper-"${VERSION}" supports dynamic reconfiguration, enabling it
+    echo "standaloneEnabled=false" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg
+    echo "dynamicConfigFile="${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg.dynamic" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg
+fi
+
+# TODO: This is a hack, netcat is convenient to have in the zookeeper container
+# I want to avoid using a custom zookeeper image just for this. So copy it.
+NC=$(which nc)
+if [ "${NC}" != "" ]; then
+    echo copying nc into "${INSTALL_VOLUME}"
+    cp "${NC}" "${INSTALL_VOLUME}"
+fi
--- a/test/images/pets/zookeeper/on-start.sh
+++ b/test/images/pets/zookeeper/on-start.sh
@@ -0,0 +1,106 @@
+#! /bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+# This script configures zookeeper cluster member ship for version of zookeeper
+# >= 3.5.0. It should not be used with the on-change.sh script in this example.
+# As of April-2016 is 3.4.8 is the latest stable.
+
+# Both /opt and /tmp/zookeeper are assumed to be volumes shared with the parent.
+# The format of each line in the dynamic config file is:
+# server.<1 based index>=<server-dns-name>:<peer port>:<election port>[:role];[<client port address>:]<client port>
+# <1 based index> is the server index that matches the id in datadir/myid
+# <peer port> is the port on which peers communicate to agree on updates
+# <election port> is the port used for leader election
+# [:role] can be set to observer, participant by default
+# <client port address> is optional and defaults to 0.0.0.0
+# <client port> is the port on which the server accepts client connections
+
+CFG=/opt/zookeeper/conf/zoo.cfg.dynamic
+CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak
+MY_ID_FILE=/tmp/zookeeper/myid
+HOSTNAME=$(hostname)
+
+while read -ra LINE; do
+    PEERS=("${PEERS[@]}" $LINE)
+done
+
+# Don't add the first member as an observer
+if [ ${#PEERS[@]} -eq 1 ]; then
+    # We need to write our index in this list of servers into MY_ID_FILE.
+    # Note that this may not always coincide with the hostname id.
+    echo 1 > "${MY_ID_FILE}"
+    echo "server.1=${PEERS[0]}:2888:3888;2181" > "${CFG}"
+    # TODO: zkServer-initialize is the safe way to handle changes to datadir
+    # because simply starting will create a new datadir, BUT if the user changed
+    # pod template they might end up with 2 datadirs and brief split brain.
+    exit
+fi
+
+# Every subsequent member is added as an observer and promoted to a participant
+echo "" > "${CFG_BAK}"
+i=0
+LEADER=$HOSTNAME
+for peer in "${PEERS[@]}"; do
+    let i=i+1
+    if [[ "${peer}" == *"${HOSTNAME}"* ]]; then
+      MY_ID=$i
+      MY_NAME=${peer}
+      echo $i > "${MY_ID_FILE}"
+      echo "server.${i}=${peer}:2888:3888:observer;2181" >> "${CFG_BAK}"
+    else
+      if [[ $(echo srvr | /opt/nc "${peer}" 2181 | grep Mode) = "Mode: leader" ]]; then
+        LEADER="${peer}"
+      fi
+      echo "server.${i}=${peer}:2888:3888:participant;2181" >> "${CFG_BAK}"
+    fi
+done
+
+# zookeeper won't start without myid anyway.
+# This means our hostname wasn't in the peer list.
+if [ ! -f "${MY_ID_FILE}" ]; then
+  exit 1
+fi
+
+# Once the dynamic config file is written it shouldn't be modified, so the final
+# reconfigure needs to happen through the "reconfig" command.
+cp ${CFG_BAK} ${CFG}
+
+# TODO: zkServer-initialize is the safe way to handle changes to datadir
+# because simply starting will create a new datadir, BUT if the user changed
+# pod template they might end up with 2 datadirs and brief split brain.
+/opt/zookeeper/bin/zkServer.sh start
+
+# TODO: We shouldn't need to specify the address of the master as long as
+# there's quorum. According to the docs the new server is just not allowed to
+# vote, it's still allowed to propose config changes, and it knows the
+# existing members of the ensemble from *its* config.
+ADD_SERVER="server.$MY_ID=$MY_NAME:2888:3888:participant;0.0.0.0:2181"
+/opt/zookeeper/bin/zkCli.sh reconfig -s "${LEADER}":2181 -add "${ADD_SERVER}"
+
+# Prove that we've actually joined the running cluster
+ITERATION=0
+until $(echo config | /opt/nc localhost 2181 | grep "${ADD_SERVER}" > /dev/null); do
+  echo $ITERATION] waiting for updated config to sync back to localhost
+  sleep 1
+  let ITERATION=ITERATION+1
+  if [ $ITERATION -eq 20 ]; then
+    exit 1
+  fi
+done
+
+/opt/zookeeper/bin/zkServer.sh stop