Ensure pods both running and ready before starting e2e tests

This commit is contained in:
Max Forbes
2015-05-18 13:49:32 -07:00
parent b963307496
commit 1da46ca3d0
9 changed files with 123 additions and 79 deletions

View File

@@ -21,7 +21,7 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}
: ${KUBECTL:=${KUBE_VERSION_ROOT}/cluster/kubectl.sh}
: ${KUBE_CONFIG_FILE:="config-test.sh"}
export KUBECTL KUBE_CONFIG_FILE

View File

@@ -21,7 +21,7 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}
: ${KUBECTL:=${KUBE_VERSION_ROOT}/cluster/kubectl.sh}
: ${KUBE_CONFIG_FILE:="config-test.sh"}
export KUBECTL KUBE_CONFIG_FILE

View File

@@ -21,7 +21,7 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}
: ${KUBECTL:=${KUBE_VERSION_ROOT}/cluster/kubectl.sh}
: ${KUBE_CONFIG_FILE:="config-test.sh"}
export KUBECTL KUBE_CONFIG_FILE

View File

@@ -21,7 +21,7 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}
: ${KUBECTL:=${KUBE_VERSION_ROOT}/cluster/kubectl.sh}
: ${KUBE_CONFIG_FILE:="config-test.sh"}
export KUBECTL KUBE_CONFIG_FILE

View File

@@ -14,8 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# e2e-status checks that the status of a cluster is acceptable for running
# e2e tests.
set -o errexit
set -o nounset
set -o pipefail
@@ -23,7 +21,7 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}
: ${KUBECTL:=${KUBE_VERSION_ROOT}/cluster/kubectl.sh}
: ${KUBE_CONFIG_FILE:="config-test.sh"}
export KUBECTL KUBE_CONFIG_FILE
@@ -34,49 +32,3 @@ source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
prepare-e2e
${KUBECTL} version
# Before running tests, ensure that all pods are 'Running'. Tests can timeout
# and fail because the test pods don't run in time. The problem is that the pods
# that a cluster runs on startup take too long to start running, with sequential
# Docker pulls of large images being the culprit. These startup pods block the
# test pods from running.
# Settings:
# timeout is in seconds; 1200 = 20 minutes.
timeout=1200
# pause is how many seconds to sleep between pod get calls.
pause=5
# min_pods is the minimum number of pods we require.
min_pods=1
# Check pod statuses.
deadline=$(($(date '+%s')+${timeout}))
echo "Waiting at most ${timeout} seconds for all pods to be 'Running'" >&2
all_running=0
until [[ ${all_running} == 1 ]]; do
if [[ "$(date '+%s')" -ge "${deadline}" ]]; then
echo "All pods never 'Running' in time." >&2
exit 1
fi
statuses=($(${KUBECTL} get pods --template='{{range.items}}{{.status.phase}} {{end}}' --api-version=v1beta3))
# Ensure that we have enough pods.
echo "Found ${#statuses[@]} pods with statuses: ${statuses[@]}" >&2
if [[ ${#statuses[@]} -lt ${min_pods} ]]; then
continue
fi
# Then, ensure all pods found are 'Running'.
found_running=1
for status in "${statuses[@]}"; do
if [[ "${status}" != "Running" ]]; then
# If we find a pod that isn't 'Running', sleep here to avoid delaying
# other code paths (where all pods are 'Running').
found_running=0
sleep ${pause}
break
fi
done
all_running=${found_running}
done
echo "All pods are 'Running'" >&2

View File

@@ -21,7 +21,7 @@ set -o pipefail
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}
: ${KUBECTL:=${KUBE_VERSION_ROOT}/cluster/kubectl.sh}
: ${KUBE_CONFIG_FILE:="config-test.sh"}
export KUBECTL KUBE_CONFIG_FILE