diff --git a/test/kubemark/common.sh b/test/kubemark/common.sh index 52e9c8056e6..51561ca8c5b 100644 --- a/test/kubemark/common.sh +++ b/test/kubemark/common.sh @@ -28,6 +28,8 @@ EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" RETRIES=3 export KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" +export KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" +export RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" # Runs gcloud compute command with the given parameters. Up to $RETRIES will be made # to execute the command. diff --git a/test/kubemark/resources/heapster_template.json b/test/kubemark/resources/heapster_template.json new file mode 100644 index 00000000000..b88ad74510c --- /dev/null +++ b/test/kubemark/resources/heapster_template.json @@ -0,0 +1,83 @@ +{ + "kind": "ReplicationController", + "apiVersion": "v1", + "metadata": { + "name": "heapster-v1.1.0.beta1", + "labels": { + "k8s-app": "heapster", + "version": "v1.1.0.beta1" + } + }, + "spec": { + "replicas": 1, + "selector": { + "k8s-app": "heapster", + "version": "v1.1.0.beta1" + }, + "template": { + "metadata": { + "labels": { + "k8s-app": "heapster", + "version": "v1.1.0.beta1" + } + }, + "spec": { + "volumes": [ + { + "name": "kubeconfig-volume", + "secret": { + "secretName": "kubeconfig" + } + } + ], + "containers": [ + { + "name": "heapster", + "image": "gcr.io/google_containers/heapster:v1.1.0-beta1", + "resources": { + "requests": { + "cpu": "100m", + "memory": "##METRICS_MEM##Mi" + } + }, + "command": [ + "/heapster" + ], + "args": [ + "--source=kubernetes:https://##MASTER_IP##:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/kubeconfig" + ], + "volumeMounts": [ + { + "name": "kubeconfig-volume", + "mountPath": "/kubeconfig" + } + ] + }, + { + "name": "eventer", + "image": "gcr.io/google_containers/heapster:v1.1.0-beta1", + "resources": { + "requests": { + "cpu": "100m", + "memory": "##EVENTER_MEM##Ki" + } + }, + "command": [ + "/eventer" + ], + "args": [ + "--source=kubernetes:https://104.197.233.84:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/kubeconfig" + ], + "volumeMounts": [ + { + "name": "kubeconfig-volume", + "mountPath": "/kubeconfig" + } + ] + + }] + } + } + } +} + diff --git a/test/kubemark/hollow-node_template.json b/test/kubemark/resources/hollow-node_template.json similarity index 100% rename from test/kubemark/hollow-node_template.json rename to test/kubemark/resources/hollow-node_template.json diff --git a/test/kubemark/kubemark-ns.json b/test/kubemark/resources/kubemark-ns.json similarity index 100% rename from test/kubemark/kubemark-ns.json rename to test/kubemark/resources/kubemark-ns.json diff --git a/test/kubemark/run-e2e-tests.sh b/test/kubemark/run-e2e-tests.sh index 16ccd4f4680..42415b7964b 100755 --- a/test/kubemark/run-e2e-tests.sh +++ b/test/kubemark/run-e2e-tests.sh @@ -30,7 +30,7 @@ echo "Kubemark master name: ${MASTER_NAME}" detect-master export KUBE_MASTER_URL="https://${KUBE_MASTER_IP}" -export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/kubeconfig.loc" +export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/resources/kubeconfig.loc" export E2E_MIN_STARTUP_PODS=0 if [[ -z "$@" ]]; then diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 1391627d2c1..b872d2599e0 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -104,9 +104,9 @@ create-certs ${MASTER_IP} KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -echo "${CA_CERT_BASE64}" | base64 -d > ca.crt -echo "${KUBECFG_CERT_BASE64}" | base64 -d > kubecfg.crt -echo "${KUBECFG_KEY_BASE64}" | base64 -d > kubecfg.key +echo "${CA_CERT_BASE64}" | base64 -d > "${RESOURCE_DIRECTORY}/ca.crt" +echo "${KUBECFG_CERT_BASE64}" | base64 -d > "${RESOURCE_DIRECTORY}/kubecfg.crt" +echo "${KUBECFG_KEY_BASE64}" | base64 -d > "${RESOURCE_DIRECTORY}/kubecfg.key" until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" --command="ls" &> /dev/null; do sleep 1 @@ -129,14 +129,14 @@ gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \ if [ "${RUN_FROM_DISTRO}" == "false" ]; then gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \ "${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz" \ - "${KUBE_ROOT}/test/kubemark/start-kubemark-master.sh" \ - "${KUBE_ROOT}/test/kubemark/configure-kubectl.sh" \ + "${KUBEMARK_DIRECTORY}/start-kubemark-master.sh" \ + "${KUBEMARK_DIRECTORY}/configure-kubectl.sh" \ "${MASTER_NAME}":~ else gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \ "${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz" \ - "${KUBE_ROOT}/test/kubemark/start-kubemark-master.sh" \ - "${KUBE_ROOT}/test/kubemark/configure-kubectl.sh" \ + "${KUBEMARK_DIRECTORY}/start-kubemark-master.sh" \ + "${KUBEMARK_DIRECTORY}/configure-kubectl.sh" \ "${MASTER_NAME}":~ fi @@ -163,7 +163,7 @@ contexts: name: kubemark-context current-context: kubemark-context" | base64 | tr -d "\n\r") -KUBECONFIG_SECRET=kubeconfig_secret.json +KUBECONFIG_SECRET="${RESOURCE_DIRECTORY}/kubeconfig_secret.json" cat > "${KUBECONFIG_SECRET}" << EOF { "apiVersion": "v1", @@ -178,7 +178,7 @@ cat > "${KUBECONFIG_SECRET}" << EOF } EOF -LOCAL_KUBECONFIG=${KUBE_ROOT}/test/kubemark/kubeconfig.loc +LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.loc" cat > "${LOCAL_KUBECONFIG}" << EOF apiVersion: v1 kind: Config @@ -202,17 +202,30 @@ contexts: current-context: kubemark-context EOF -sed "s/##numreplicas##/${NUM_NODES:-10}/g" "${KUBE_ROOT}"/test/kubemark/hollow-node_template.json > "${KUBE_ROOT}"/test/kubemark/hollow-node.json -sed -i'' -e "s/##project##/${PROJECT}/g" "${KUBE_ROOT}"/test/kubemark/hollow-node.json -"${KUBECTL}" create -f "${KUBE_ROOT}"/test/kubemark/kubemark-ns.json + +sed "s/##numreplicas##/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" +sed -i'' -e "s/##project##/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" + +mkdir "${RESOURCE_DIRECTORY}/addons" || true + +sed "s/##MASTER_IP##/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" +metrics_mem_per_node=4 +metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES:-10})) +sed -i'' -e "s/##METRICS_MEM##/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" +eventer_mem_per_node=500 +eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10})) +sed -i'' -e "s/##EVENTER_MEM##/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" + +"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" "${KUBECTL}" create -f "${KUBECONFIG_SECRET}" --namespace="kubemark" -"${KUBECTL}" create -f "${KUBE_ROOT}"/test/kubemark/hollow-node.json --namespace="kubemark" +"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" +"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark" rm "${KUBECONFIG_SECRET}" echo "Waiting for all HollowNodes to become Running..." start=$(date +%s) -nodes=$("${KUBECTL}" --kubeconfig="${KUBE_ROOT}"/test/kubemark/kubeconfig.loc get node) || true +nodes=$("${KUBECTL}" --kubeconfig="${RESOURCE_DIRECTORY}/kubeconfig.loc" get node) || true ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) until [[ "${ready}" -ge "${NUM_NODES}" ]]; do @@ -224,16 +237,16 @@ until [[ "${ready}" -ge "${NUM_NODES}" ]]; do echo "" echo "Timeout waiting for all HollowNodes to become Running" # Try listing nodes again - if it fails it means that API server is not responding - if "${KUBECTL}" --kubeconfig="${KUBE_ROOT}"/test/kubemark/kubeconfig.loc get node &> /dev/null; then + if "${KUBECTL}" --kubeconfig="${RESOURCE_DIRECTORY}/kubeconfig.loc" get node &> /dev/null; then echo "Found only ${ready} ready Nodes while waiting for ${NUM_NODES}." exit 1 fi echo "Got error while trying to list Nodes. Probably API server is down." exit 1 fi - nodes=$("${KUBECTL}" --kubeconfig="${KUBE_ROOT}"/test/kubemark/kubeconfig.loc get node) || true + nodes=$("${KUBECTL}" --kubeconfig="${RESOURCE_DIRECTORY}/kubeconfig.loc" get node) || true ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) done - echo "" + echo "Password to kubemark master: ${password}" diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh index da3e6973d74..d4be78089e9 100755 --- a/test/kubemark/stop-kubemark.sh +++ b/test/kubemark/stop-kubemark.sh @@ -19,8 +19,10 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/test/kubemark/common.sh" -"${KUBECTL}" delete -f ${KUBE_ROOT}/test/kubemark/hollow-kubelet.json &> /dev/null || true -"${KUBECTL}" delete -f ${KUBE_ROOT}/test/kubemark/kubemark-ns.json &> /dev/null || true +"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-kubelet.json" &> /dev/null || true +"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true +"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true +rm -rf "${RESOURCE_DIRECTORY}/addons" GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet" @@ -42,5 +44,5 @@ if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then ${GCLOUD_COMMON_ARGS} || true fi -rm -rf "${KUBE_ROOT}/test/kubemark/kubeconfig.loc" &> /dev/null || true -rm "ca.crt" "kubecfg.crt" "kubecfg.key" "${KUBE_ROOT}/test/kubemark/hollow-node.json" &> /dev/null || true +rm -rf "${RESOURCE_DIRECTORY}/addons" "${RESOURCE_DIRECTORY}/kubeconfig.loc" &> /dev/null || true +rm "${RESOURCE_DIRECTORY}/ca.crt" "${RESOURCE_DIRECTORY}/kubecfg.crt" "${RESOURCE_DIRECTORY}/kubecfg.key" "${RESOURCE_DIRECTORY}/hollow-node.json" &> /dev/null || true