Add integration test framework

Signed-off-by: Lantao Liu <lantaol@google.com>

hack/test-cri.sh

@@ -22,6 +22,7 @@ source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
 FOCUS=${FOCUS:-}
 # SKIP skips the test to skip.
 SKIP=${SKIP:-""}
+# REPORT_DIR is the the directory to store test logs.
 REPORT_DIR=${REPORT_DIR:-"/tmp/test-cri"}
 
 # Check GOPATH
@@ -35,7 +36,6 @@ GOPATH=${GOPATH%%:*}
 
 CRITEST=${GOPATH}/bin/critest
 CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools
-CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
 
 # Install critest
 if [ ! -x "$(command -v ${CRITEST})" ]; then
@@ -48,12 +48,12 @@ fi
 which ${CRITEST}
 
 mkdir -p ${REPORT_DIR}
-start_cri_containerd ${REPORT_DIR}
+test_setup ${REPORT_DIR}
 
 # Run cri validation test
 sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\"" validation
 test_exit_code=$?
 
-kill_cri_containerd
+test_teardown
 
 exit ${test_exit_code}

hack/test-e2e-node.sh

@@ -26,7 +26,9 @@ DEFAULT_SKIP+="|ImageID"
 export FOCUS=${FOCUS:-""}
 # SKIP skips the test to skip.
 export SKIP=${SKIP:-${DEFAULT_SKIP}}
+# REPORT_DIR is the the directory to store test logs.
 REPORT_DIR=${REPORT_DIR:-"/tmp/test-e2e-node"}
+# UPLOAD_LOG indicates whether to upload test log to gcs.
 UPLOAD_LOG=${UPLOAD_LOG:-false}
 
 # Check GOPATH
@@ -67,18 +69,18 @@ git fetch --all
 git checkout ${KUBERNETES_VERSION}
 
 mkdir -p ${REPORT_DIR}
-start_cri_containerd ${REPORT_DIR}
+test_setup ${REPORT_DIR}
 
 make test-e2e-node \
 	RUNTIME=remote \
-	CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/cri-containerd.sock \
+	CONTAINER_RUNTIME_ENDPOINT=unix://${CRICONTAINERD_SOCK} \
 	ARTIFACTS=${REPORT_DIR} \
 	TEST_ARGS='--kubelet-flags=--cgroups-per-qos=true \
 	--kubelet-flags=--cgroup-root=/ \
 	--prepull-images=false'
 test_exit_code=$?
 
-kill_cri_containerd
+test_teardown
 
 sudo iptables-restore < ${ORIGINAL_RULES}
 rm ${ORIGINAL_RULES}

hack/test-integration.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -o nounset
+set -o pipefail
+
+source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh
+cd ${ROOT}
+
+# FOCUS focuses the test to run.
+FOCUS=${FOCUS:-""}
+# REPORT_DIR is the the directory to store test logs.
+REPORT_DIR=${REPORT_DIR:-"/tmp/test-integration"}
+
+mkdir -p ${REPORT_DIR}
+test_setup ${REPORT_DIR}
+
+# Run integration test.
+sudo ${ROOT}/_output/integration.test --test.run="${FOCUS}" --test.v
+test_exit_code=$?
+
+test_teardown
+
+exit ${test_exit_code}

hack/test-utils.sh

@@ -19,8 +19,10 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
 # CRI_CONTAINERD_FLAGS are the extra flags to use when start cri-containerd.
 CRI_CONTAINERD_FLAGS=${CRI_CONTAINERD_FLAGS:-""}
 
-# start_cri_containerd starts containerd and cri-containerd.
-start_cri_containerd() {
+CRICONTAINERD_SOCK=/var/run/cri-containerd.sock
+
+# test_setup starts containerd and cri-containerd.
+test_setup() {
   local report_dir=$1 
   if [ ! -x ${ROOT}/_output/cri-containerd ]; then
     echo "cri-containerd is not built"
@@ -32,29 +34,35 @@ start_cri_containerd() {
     echo "containerd is not installed, please run hack/install-deps.sh"
     exit 1
   fi
-  kill_cri_containerd
-  sudo containerd -l debug &> ${report_dir}/containerd.log &
-
+  sudo pkill containerd
+  sudo containerd &> ${report_dir}/containerd.log &
   # Wait for containerd to be running by using the containerd client ctr to check the version
   # of the containerd server. Wait an increasing amount of time after each of five attempts
-  local MAX_ATTEMPTS=5
-  local attempt_num=1
-  until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
-  do
-      echo "attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..."
-      sleep $(( attempt_num++ ))
-  done
+  readiness_check "sudo ctr version"
 
   # Start cri-containerd
   sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 ${CRI_CONTAINERD_FLAGS} \
 	  &> ${report_dir}/cri-containerd.log &
+  readiness_check "sudo ${GOPATH}/bin/crictl --runtime-endpoint=${CRICONTAINERD_SOCK} info"
 }
 
-# kill_cri_containerd kills containerd and cri-containerd.
-kill_cri_containerd() {
+# test_teardown kills containerd and cri-containerd.
+test_teardown() {
   sudo pkill containerd
 }
 
+# readiness_check checks readiness of a daemon with specified command.
+readiness_check() {
+  local command=$1
+  local MAX_ATTEMPTS=5
+  local attempt_num=1
+  until ${command} &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
+  do
+      echo "$attempt_num attempt \"$command\"! Trying again in $attempt_num seconds..."
+      sleep $(( attempt_num++ ))
+  done
+}
+
 # upload_logs_to_gcs uploads test logs to gcs.
 # Var set:
 # 1. Bucket: gcs bucket to upload logs.