diff --git a/cluster/mesos/docker/.gitignore b/cluster/mesos/docker/.gitignore new file mode 100644 index 00000000000..b2290143a47 --- /dev/null +++ b/cluster/mesos/docker/.gitignore @@ -0,0 +1 @@ +certs diff --git a/cluster/mesos/docker/common/bin/await-file b/cluster/mesos/docker/common/bin/await-file new file mode 100755 index 00000000000..1ac9f7a3a75 --- /dev/null +++ b/cluster/mesos/docker/common/bin/await-file @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wait for a file to exist in the filesystem. +# Block up to the timeout duration in seconds (default: 10). +# Usage: await-health-check [-t=]
+# Requires: timeout, file + +set -o errexit +set -o nounset +set -o pipefail + +duration=10 +if [[ "${1:-}" == "-t="* ]]; then + duration="${1:3}" + [ -z "${duration}" ] && echo "Invalid duration supplied" && exit 1 + shift +fi + +file="$1" +[ -z "$file" ] && echo "No file supplied" && exit 1 + +echo "Waiting (up to ${duration}s) for ${file} to exist" +if ! timeout "${duration}" bash -c "while [ ! -f \"${file}\" ]; do sleep 0.5; done"; then + echo "Timed out" + exit 1 +fi + +echo "File ${file} exists now!" diff --git a/cluster/mesos/docker/common/bin/await-health-check b/cluster/mesos/docker/common/bin/await-health-check new file mode 100755 index 00000000000..577f5787ac0 --- /dev/null +++ b/cluster/mesos/docker/common/bin/await-health-check @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wait for a service to accept connections. +# Block up to the timeout duration in seconds (default: 10). +# Usage: await-health-check [-t=]
+# Requires: timeout, health-check + +set -o errexit +set -o nounset +set -o pipefail + +duration=10 +if [[ "${1:-}" == "-t="* ]]; then + duration="${1:3}" + [ -z "${duration}" ] && echo "Invalid duration supplied" && exit 1 + shift +fi + +address=${1:-} +[ -z "${address}" ] && echo "No address supplied" && exit 1 + +bin=$(cd $(dirname $0) && pwd -P) + +echo "Waiting (up to ${duration}s) for ${address} to be healthy" +if ! timeout "${duration}" bash -c "while ! ${bin}/health-check ${address}; do sleep 0.5; done"; then + echo "Timed out" + exit 1 +fi + +echo "Health check of ${address} succeeded!" diff --git a/cluster/mesos/docker/common/bin/health-check b/cluster/mesos/docker/common/bin/health-check new file mode 100755 index 00000000000..3b90c0a92b1 --- /dev/null +++ b/cluster/mesos/docker/common/bin/health-check @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Curl an endpoint and expect it to respond with status 200. +# Usage: health-check
+ +set -o errexit +set -o nounset +set -o pipefail + +address=${1:-} +[ -z "${address}" ] && echo "No address supplied" && exit 1 + +status=$(curl -s -o /dev/null -w '%{http_code}' ${address}) +if [[ "${status}" == '200' ]]; then + exit 0 +fi +if [[ "${status}" == '000' ]]; then + exit 7 +fi + +exit 1 diff --git a/cluster/mesos/docker/common/bin/resolveip b/cluster/mesos/docker/common/bin/resolveip new file mode 100755 index 00000000000..334884c1285 --- /dev/null +++ b/cluster/mesos/docker/common/bin/resolveip @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Resolve an IP from a hostname (using getent) +# Usage: resolveip +# Requires: getent +# TODO: Mac support + +set -o errexit +set -o nounset +set -o pipefail + +hostname=$1 +[ -z "${hostname}" ] && echo "No hostname supplied" && exit 1 + +getent hosts "${hostname}" | cut -d' ' -f1 | sort -u | tail -1 diff --git a/cluster/mesos/docker/config-default.sh b/cluster/mesos/docker/config-default.sh new file mode 100755 index 00000000000..1a1fd24c138 --- /dev/null +++ b/cluster/mesos/docker/config-default.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Contains configuration values for interacting with the mesos/docker cluster + +NUM_MINIONS=${NUM_MINIONS:-2} +INSTANCE_PREFIX="${INSTANCE_PREFIX:-kubernetes}" +MASTER_NAME="${INSTANCE_PREFIX}-master" +MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) + +SERVICE_CLUSTER_IP_RANGE=10.10.10.0/24 + +# Extra options to set on the Docker command line. This is useful for setting +# --insecure-registry for local registries. +DOCKER_OPTS="" + +# Optional: Deploy cluster DNS. +#ENABLE_CLUSTER_DNS=false +ENABLE_CLUSTER_DNS=true +DNS_SERVER_IP="10.10.10.10" +DNS_DOMAIN="cluster.local" +DNS_REPLICAS=1 + +# Optional: Deploy cluster web interface. +ENABLE_CLUSTER_UI=true diff --git a/cluster/mesos/docker/config-test.sh b/cluster/mesos/docker/config-test.sh new file mode 100644 index 00000000000..4e6261bb4bd --- /dev/null +++ b/cluster/mesos/docker/config-test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Contains configuration values for interacting with the docker-compose cluster in test mode +#Set NUM_MINIONS to minimum required for testing. +NUM_MINIONS=2 + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../.. +source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/config-default.sh" diff --git a/cluster/mesos/docker/deploy-addons.sh b/cluster/mesos/docker/deploy-addons.sh new file mode 100755 index 00000000000..30e803c62b3 --- /dev/null +++ b/cluster/mesos/docker/deploy-addons.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deploy the addon services after the cluster is available +# TODO: integrate with or use /cluster/saltbase/salt/kube-addons/kube-addons.sh +# Requires: +# ENABLE_CLUSTER_DNS (Optional) - 'Y' to deploy kube-dns +# KUBE_SERVER (Optional) - url to the api server for configuring kube-dns + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) +source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/${KUBE_CONFIG_FILE-"config-default.sh"}" +source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util-temp-dir.sh" +kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + + +function deploy_dns { + echo "Deploying DNS Addon" 1>&2 + local workspace=$(pwd) + + # Process salt pillar templates manually + sed -e "s/{{ pillar\['dns_replicas'\] }}/${DNS_REPLICAS}/g;s/{{ pillar\['dns_domain'\] }}/${DNS_DOMAIN}/g" "${KUBE_ROOT}/cluster/addons/dns/skydns-rc.yaml.in" > "${workspace}/skydns-rc.yaml" + sed -e "s/{{ pillar\['dns_server'\] }}/${DNS_SERVER_IP}/g" "${KUBE_ROOT}/cluster/addons/dns/skydns-svc.yaml.in" > "${workspace}/skydns-svc.yaml" + + # Use kubectl to create skydns rc and service + "${kubectl}" create -f "${workspace}/skydns-rc.yaml" + "${kubectl}" create -f "${workspace}/skydns-svc.yaml" +} + +function deploy_ui { + echo "Deploying UI Addon" 1>&2 + + # Use kubectl to create ui rc and service + "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/kube-ui/kube-ui-rc.yaml" + "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/kube-ui/kube-ui-svc.yaml" +} + +# create the kube-system namespace +"${kubectl}" create -f "${KUBE_ROOT}/cluster/mesos/docker/kube-system-ns.yaml" + +if [ "${ENABLE_CLUSTER_DNS}" == true ]; then + cluster::mesos::docker::run_in_temp_dir 'k8sm-dns' 'deploy_dns' +fi + +if [ "${ENABLE_CLUSTER_UI}" == true ]; then + deploy_ui +fi diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml new file mode 100644 index 00000000000..349b2a2a5a9 --- /dev/null +++ b/cluster/mesos/docker/docker-compose.yml @@ -0,0 +1,157 @@ +ambassador: + image: cpuguy83/docker-grand-ambassador:0.9.1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: "-name docker_apiserver_1" +etcd: + hostname: etcd + image: quay.io/coreos/etcd:v2.0.12 + ports: [ "4001:4001" ] + command: > + --listen-client-urls 'http://etcd:4001' + --advertise-client-urls 'http://etcd:4001' + --initial-cluster-state new +mesosmaster1: + hostname: mesosmaster1 + image: mesosphere/mesos:0.22.0-1.0.ubuntu1404 + entrypoint: [ "mesos-master" ] + ports: [ "5050:5050" ] + environment: + - MESOS_HOSTNAME=mesosmaster1 + - MESOS_PORT=5050 + - MESOS_LOG_DIR=/var/log/mesos + - MESOS_QUORUM=1 + - MESOS_REGISTRY=in_memory + - MESOS_WORK_DIR=/var/lib/mesos + links: + - etcd + - "ambassador:apiserver" +mesosslave1: + hostname: mesosslave1 + privileged: true + image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404 + entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave1 | cut -d' ' -f1 | sort -u | tail -1)" ] + command: ~ + environment: + - MESOS_MASTER=mesosmaster1:5050 + - MESOS_PORT=5051 + - MESOS_LOG_DIR=/var/log/mesos + - MESOS_LOGGING_LEVEL=INFO + - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099] + - MESOS_SWITCH_USER=0 + - MESOS_CONTAINERIZERS=docker,mesos + - DOCKER_NETWORK_OFFSET=0.0.1.0 + - DOCKER_DAEMON_ARGS=--log-level=error + links: + - etcd + - mesosmaster1 + - "ambassador:apiserver" + volumes: + - /var/tmp/mesosslave1:/var/lib/docker +mesosslave2: + hostname: mesosslave2 + privileged: true + image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404 + entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave2 | cut -d' ' -f1 | sort -u | tail -1)" ] + command: ~ + environment: + - MESOS_MASTER=mesosmaster1:5050 + - MESOS_PORT=5051 + - MESOS_LOG_DIR=/var/log/mesos + - MESOS_LOGGING_LEVEL=INFO + - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099] + - MESOS_SWITCH_USER=0 + - MESOS_CONTAINERIZERS=docker,mesos + - DOCKER_NETWORK_OFFSET=0.0.2.0 + - DOCKER_DAEMON_ARGS=--log-level=error + links: + - etcd + - mesosmaster1 + - "ambassador:apiserver" + volumes: + - /var/tmp/mesosslave2:/var/lib/docker +apiserver: + hostname: apiserver + image: mesosphere/kubernetes-mesos + entrypoint: + - /bin/bash + - "-c" + - > + echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" && + (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo " mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) && + await-health-check http://etcd:4001/health && + await-health-check http://mesosmaster1:5050/health && + await-file -t=60 /var/run/kubernetes/auth/apiserver.crt && + km apiserver + --address=$(resolveip apiserver) + --external-hostname=apiserver + --etcd-servers=http://etcd:4001 + --port=8888 + --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota + --authorization-mode=AlwaysAllow + --token-auth-file=/var/run/kubernetes/auth/token-users + --basic-auth-file=/var/run/kubernetes/auth/basic-users + --service-account-key-file=/var/run/kubernetes/auth/service-accounts.key + --service-cluster-ip-range=10.10.10.0/24 + --service-node-port-range=30000-32767 + --cloud-provider=mesos + --cloud-config=/opt/mesos-cloud.conf + --tls-cert-file=/var/run/kubernetes/auth/apiserver.crt + --tls-private-key-file=/var/run/kubernetes/auth/apiserver.key + --v=2 + ports: [ "8888:8888", "6443:6443" ] + volumes: + - ./certs/apiserver:/var/run/kubernetes/auth + links: + - etcd + - mesosmaster1 +controller: + hostname: controller + image: mesosphere/kubernetes-mesos + entrypoint: + - /bin/bash + - "-c" + - > + echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" && + (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo " mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) && + await-health-check -t=60 http://mesosmaster1:5050/health && + await-health-check -t=60 http://apiserver:8888/healthz && + km controller-manager + --master=http://apiserver:8888 + --cloud-config=/opt/mesos-cloud.conf + --service-account-private-key-file=/var/run/kubernetes/auth/service-accounts.key + --root-ca-file=/var/run/kubernetes/auth/root-ca.crt + --v=2 + volumes: + - ./certs/controller:/var/run/kubernetes/auth + links: + - mesosmaster1 + - apiserver +scheduler: + hostname: scheduler + image: mesosphere/kubernetes-mesos + entrypoint: + - /bin/bash + - "-c" + - > + echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" && + (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo " mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) && + await-health-check http://etcd:4001/health && + await-health-check http://mesosmaster1:5050/health && + await-health-check -t=60 http://apiserver:8888/healthz && + km scheduler + --address=$(resolveip scheduler) + --hostname-override=scheduler + --etcd-servers=http://etcd:4001 + --mesos-user=root + --api-servers=http://apiserver:8888 + --mesos-master=mesosmaster1:5050 + --cluster-dns=10.10.10.10 + --cluster-domain=cluster.local + --v=2 + links: + - etcd + - mesosmaster1 + - mesosslave1 + - mesosslave2 + - apiserver diff --git a/cluster/mesos/docker/km/Dockerfile b/cluster/mesos/docker/km/Dockerfile new file mode 100644 index 00000000000..6f170c0a66e --- /dev/null +++ b/cluster/mesos/docker/km/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:14.04.2 +MAINTAINER Mesosphere + +RUN locale-gen en_US.UTF-8 +RUN dpkg-reconfigure locales +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +RUN apt-get update -qq && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \ + wget \ + curl \ + && \ + apt-get clean + +COPY ./bin/* /usr/local/bin/ +ADD ./opt/mesos-cloud.conf /opt/ diff --git a/cluster/mesos/docker/km/build.sh b/cluster/mesos/docker/km/build.sh new file mode 100755 index 00000000000..984fcfc27c2 --- /dev/null +++ b/cluster/mesos/docker/km/build.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds a docker image that contains the kubernetes-mesos binaries. + +set -o errexit +set -o nounset +set -o pipefail + +IMAGE_REPO=${IMAGE_REPO:-mesosphere/kubernetes-mesos} +IMAGE_TAG=${IMAGE_TAG:-latest} + +script_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P) +KUBE_ROOT=$(cd ${script_dir}/../../../.. && pwd -P) + +# Find a platform specific binary, whether it was cross compiled, locally built, or downloaded. +find-binary() { + local lookfor="${1}" + local platform="${2}" + local locations=( + "${KUBE_ROOT}/_output/dockerized/bin/${platform}/${lookfor}" + "${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}" + "${KUBE_ROOT}/platforms/${platform}/${lookfor}" + ) + local bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 ) + echo -n "${bin}" +} + +km_path=$(find-binary km linux/amd64) +if [ -z "$km_path" ]; then + echo "Failed to find km binary" 1>&2 + exit 1 +fi +kube_bin_path=$(dirname ${km_path}) +common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P) + +cd "${KUBE_ROOT}" + +# create temp workspace to place compiled binaries with image-specific scripts +# create temp workspace dir in KUBE_ROOT to avoid permission issues of TMPDIR on mac os x +workspace=$(env TMPDIR=$PWD mktemp -d -t "k8sm-workspace-XXXXXX") +echo "Workspace created: ${workspace}" + +cleanup() { + rm -rf "${workspace}" + echo "Workspace deleted: ${workspace}" +} +trap 'cleanup' EXIT + +# setup workspace to mirror script dir (dockerfile expects /bin & /opt) +echo "Copying files to workspace" + +# binaries & scripts +mkdir -p "${workspace}/bin" +#cp "${script_dir}/bin/"* "${workspace}/bin/" +cp "${common_bin_path}/"* "${workspace}/bin/" +cp "${kube_bin_path}/"* "${workspace}/bin/" + +# config +mkdir -p "${workspace}/opt" +cp "${script_dir}/opt/"* "${workspace}/opt/" + +# docker +cp "${script_dir}/Dockerfile" "${workspace}/" + +cd "${workspace}" + +# build docker image +echo "Building docker image ${IMAGE_REPO}:${IMAGE_TAG}" +set -o xtrace +docker build -t ${IMAGE_REPO}:${IMAGE_TAG} "$@" . +set +o xtrace +echo "Built docker image ${IMAGE_REPO}:${IMAGE_TAG}" diff --git a/cluster/mesos/docker/km/opt/mesos-cloud.conf b/cluster/mesos/docker/km/opt/mesos-cloud.conf new file mode 100644 index 00000000000..99aa88eee22 --- /dev/null +++ b/cluster/mesos/docker/km/opt/mesos-cloud.conf @@ -0,0 +1,3 @@ +[mesos-cloud] + http-client-timeout = 5s + state-cache-ttl = 20s diff --git a/cluster/mesos/docker/kube-system-ns.yaml b/cluster/mesos/docker/kube-system-ns.yaml new file mode 100644 index 00000000000..7fa10361af6 --- /dev/null +++ b/cluster/mesos/docker/kube-system-ns.yaml @@ -0,0 +1,6 @@ +kind: "Namespace" +apiVersion: "v1" +metadata: + name: "kube-system" + labels: + name: "kube-system" \ No newline at end of file diff --git a/cluster/mesos/docker/test/Dockerfile b/cluster/mesos/docker/test/Dockerfile new file mode 100644 index 00000000000..19cd78a595a --- /dev/null +++ b/cluster/mesos/docker/test/Dockerfile @@ -0,0 +1,49 @@ +FROM golang:1.4.2 +MAINTAINER Mesosphere + +# docker.io is suppossed to be in backports, but it's not there yet. +# https://github.com/docker/docker/issues/13253 +# http://docs.docker.com/installation/debian/#debian-jessie-80-64-bit +#RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" >> /etc/apt/sources.list +#RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list + +RUN apt-get update -qq && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \ + wget \ + curl \ + g++ \ + make \ + mercurial \ + git \ + rsync \ + patch \ + python \ + python-pip \ + apt-transport-https \ + && \ + apt-get clean + +# Install latest Docker +# RUN curl -sSL https://get.docker.com/ubuntu/ | sh + +# Install Docker 1.6.2 (docker 1.7 has regressions) +RUN echo deb https://get.docker.com/ubuntu docker main > /etc/apt/sources.list.d/docker.list && \ + apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \ + apt-get update -qq && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \ + lxc-docker-1.6.2 \ + && \ + apt-get clean + +RUN pip install -U docker-compose + +RUN go get github.com/tools/godep + +RUN mkdir -p /go/src/github.com/GoogleCloudPlatform/kubernetes +WORKDIR /go/src/github.com/GoogleCloudPlatform/kubernetes + +COPY ./bin/* /usr/local/bin/ + +RUN install-etcd.sh + +ENTRYPOINT [ "bash" ] diff --git a/cluster/mesos/docker/test/bin/install-etcd.sh b/cluster/mesos/docker/test/bin/install-etcd.sh new file mode 100755 index 00000000000..cdc6079fa7d --- /dev/null +++ b/cluster/mesos/docker/test/bin/install-etcd.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Installs etcd into /usr/local/bin + +set -o errexit +set -o nounset +set -o pipefail + +ETCD_VERSION=${ETCD_VERSION:-v2.0.11} + +full_name=etcd-${ETCD_VERSION}-linux-amd64 +archive_url=https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/${full_name}.tar.gz + +download_dir=/tmp/etcd-${ETCD_VERSION} + +mkdir ${download_dir} + +function cleanup { + rm -rf ${download_dir} +} + +trap cleanup EXIT + +cd ${download_dir} + +echo "Downloading etcd (${archive_url})..." +curl -s -L ${archive_url} | tar xvz + +echo "Installing etcd (/usr/local/bin/etcd)..." +mv ./${full_name}/etcd* /usr/local/bin/ diff --git a/cluster/mesos/docker/test/build.sh b/cluster/mesos/docker/test/build.sh new file mode 100755 index 00000000000..5ec3c61e1a9 --- /dev/null +++ b/cluster/mesos/docker/test/build.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +IMAGE_REPO=${IMAGE_REPO:-mesosphere/kubernetes-mesos-test} +IMAGE_TAG=${IMAGE_TAG:-latest} + +script_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P) +common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P) +KUBE_ROOT=$(cd ${script_dir}/../../../.. && pwd -P) + +cd "${KUBE_ROOT}" + +# create temp workspace to place common scripts with image-specific scripts +# create temp workspace dir in KUBE_ROOT to avoid permission issues of TMPDIR on mac os x +workspace=$(env TMPDIR=$PWD mktemp -d -t "k8sm-test-workspace-XXXXXX") +echo "Workspace created: ${workspace}" + +cleanup() { + rm -rf "${workspace}" + echo "Workspace deleted: ${workspace}" +} +trap 'cleanup' EXIT + +# setup workspace to mirror script dir (dockerfile expects /bin) +set -x +echo "Copying files to workspace" + +# binaries & scripts +mkdir -p "${workspace}/bin" +cp -a "${common_bin_path}/"* "${workspace}/bin/" +cp -a "${script_dir}/bin/"* "${workspace}/bin/" + +# docker +cp -a "${script_dir}/Dockerfile" "${workspace}/" + +cd "${workspace}" + +echo "Building docker image ${IMAGE_REPO}:${IMAGE_TAG}" +set -o xtrace +docker build -t ${IMAGE_REPO}:${IMAGE_TAG} "$@" . +set +o xtrace +echo "Built docker image ${IMAGE_REPO}:${IMAGE_TAG}" diff --git a/cluster/mesos/docker/util-ssl.sh b/cluster/mesos/docker/util-ssl.sh new file mode 100644 index 00000000000..e03b084bea1 --- /dev/null +++ b/cluster/mesos/docker/util-ssl.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sourcable SSL functions + +set -o errexit +set -o nounset +set -o pipefail + +script_dir=$(cd "$(dirname ${BASH_SOURCE})" && pwd -P) +source "${script_dir}/util-temp-dir.sh" + +function cluster::mesos::docker::find_openssl_config { + for candidate in "/etc/ssl/openssl.cnf" "/System/Library/OpenSSL/openssl.cnf"; do + if [ -f "${candidate}" ]; then + echo "${candidate}" + return 0 + fi + done + echo "ERROR: cannot find openssl.cnf" 1>&2 + return 1 +} + +function cluster::mesos::docker::create_root_certificate_authority { + local certdir="$1" + + openssl req -nodes -newkey rsa:2048 \ + -keyout "${certdir}/root-ca.key" \ + -out "${certdir}/root-ca.csr" \ + -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com" + + openssl x509 -req -days 3650 \ + -in "${certdir}/root-ca.csr" \ + -out "${certdir}/root-ca.crt" \ + -signkey "${certdir}/root-ca.key" +} + +# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names. +# Uses the current dir for scratch work. +function cluster::mesos::docker::create_apiserver_cert_inner { + local in_dir="$1" # must contain root-ca.crt & root-ca.key + local out_dir="$2" + local apiserver_ip="$3" + local service_ip="$4" + local workspace="$(pwd)" + + mkdir -p "${out_dir}" + + local OPENSSL_CNF=$(cluster::mesos::docker::find_openssl_config) + + # create apiserver key and certificate sign request + local SANS="IP:${apiserver_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" + openssl req -nodes -newkey rsa:2048 \ + -keyout "${workspace}/apiserver.key" -out "${workspace}/apiserver.csr" \ + -reqexts SAN -config <(cat "${OPENSSL_CNF}"; echo -e "[SAN]\nsubjectAltName=$SANS") \ + -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com" + + # sign with root-ca + mkdir -p ${workspace}/demoCA/newcerts + touch ${workspace}/demoCA/index.txt + echo 1000 > ${workspace}/demoCA/serial + openssl ca -cert "${in_dir}/root-ca.crt" -keyfile "${in_dir}/root-ca.key" \ + -batch -days 3650 -in "${workspace}/apiserver.csr" \ + -config <(sed 's/.*\(copy_extensions = copy\)/\1/' ${OPENSSL_CNF}) >/dev/null + + # check certificate for subjectAltName extension + if ! openssl x509 -in "${workspace}/demoCA/newcerts/1000.pem" -text -noout | grep -q kubernetes.default.svc.cluster.local; then + echo "ERROR: openssl failed to add subjectAltName extension" 1>&2 + return 1 + fi + + # write to out_dir + cp "${workspace}/demoCA/newcerts/1000.pem" "${out_dir}/apiserver.crt" + cp "${workspace}/apiserver.key" "${out_dir}/" +} + +# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names. +function cluster::mesos::docker::create_apiserver_cert { + local in_dir="$1" # must contain root-ca.crt & root-ca.key + local out_dir="$2" + local apiserver_ip="$3" + local service_ip="$4" + + cluster::mesos::docker::run_in_temp_dir "k8sm-certs" \ + "cluster::mesos::docker::create_apiserver_cert_inner" \ + "${in_dir}" "${out_dir}" "${apiserver_ip}" "${service_ip}" +} + +# Creates an rsa key (for signing service accounts). +function cluster::mesos::docker::create_rsa_key { + local key_file_path="$1" + openssl genrsa -out "${key_file_path}" 2048 +} + +# Creates a k8s token auth user file. +# See /docs/admin/authentication.md +function cluster::mesos::docker::create_token_user { + local user_name="$1" + echo "$(openssl rand -hex 32),${user_name},${user_name}" +} + +# Creates a k8s basic auth user file. +# See /docs/admin/authentication.md +function cluster::mesos::docker::create_basic_user { + local user_name="$1" + local password="$2" + echo "${password},${user_name},${user_name}" +} \ No newline at end of file diff --git a/cluster/mesos/docker/util-temp-dir.sh b/cluster/mesos/docker/util-temp-dir.sh new file mode 100644 index 00000000000..3e12a050a17 --- /dev/null +++ b/cluster/mesos/docker/util-temp-dir.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sourcable temp directory functions + +set -o errexit +set -o nounset +set -o pipefail + +# Runs the supplied command string in a temporary workspace directory. +function cluster::mesos::docker::run_in_temp_dir { + prefix="$1" + shift + cmd="$@" + + # create temp WORKSPACE dir in current dir to avoid permission issues of TMPDIR on mac os x + local -r workspace=$(env TMPDIR=$(pwd) mktemp -d -t "${prefix}-XXXXXX") + echo "Workspace created: ${workspace}" 1>&2 + + cleanup() { + rm -rf "${workspace}" + echo "Workspace deleted: ${workspace}" 1>&2 + } + trap 'cleanup' EXIT + + pushd "${workspace}" > /dev/null + (${cmd}) || return $? + popd > /dev/null + + trap - EXIT + cleanup +} \ No newline at end of file diff --git a/cluster/mesos/docker/util.sh b/cluster/mesos/docker/util.sh new file mode 100644 index 00000000000..a181aeffa1c --- /dev/null +++ b/cluster/mesos/docker/util.sh @@ -0,0 +1,295 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Example: +# export KUBERNETES_PROVIDER=mesos/docker +# go run hack/e2e.go -v -up -check_cluster_size=false +# go run hack/e2e.go -v -test -check_version_skew=false +# go run hack/e2e.go -v -down + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) +provider_root="${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}" + +source "${provider_root}/${KUBE_CONFIG_FILE-"config-default.sh"}" +source "${KUBE_ROOT}/cluster/common.sh" +source "${provider_root}/util-ssl.sh" + + +# Run kubernetes scripts inside docker. +# This bypasses the need to set up network routing when running docker in a VM (e.g. boot2docker). +# Trap signals and kills the docker container for better signal handing +function cluster::mesos::docker::run_in_docker { + entrypoint="$1" + if [[ "${entrypoint}" = "./"* ]]; then + # relative to project root + entrypoint="/go/src/github.com/GoogleCloudPlatform/kubernetes/${entrypoint}" + fi + shift + args="$@" + + container_id=$( + docker run \ + -d \ + -e "KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER}" \ + -v "${KUBE_ROOT}:/go/src/github.com/GoogleCloudPlatform/kubernetes" \ + -v "$(dirname ${KUBECONFIG}):/root/.kube" \ + -v "/var/run/docker.sock:/var/run/docker.sock" \ + --link docker_mesosmaster1_1:mesosmaster1 \ + --link docker_mesosslave1_1:mesosslave1 \ + --link docker_mesosslave2_1:mesosslave2 \ + --link docker_apiserver_1:apiserver \ + --entrypoint="${entrypoint}" \ + mesosphere/kubernetes-mesos-test \ + ${args} + ) + + docker logs -f "${container_id}" & + + # trap and kill for better signal handing + trap 'echo "Killing container ${container_id}" 1>&2 && docker kill ${container_id}' INT TERM + exit_status=$(docker wait "${container_id}") + trap - INT TERM + + if [ "$exit_status" != 0 ]; then + echo "Exited ${exit_status}" 1>&2 + fi + + docker rm -f "${container_id}" > /dev/null + + return "${exit_status}" +} + +# Generate kubeconfig data for the created cluster. +function create-kubeconfig { + local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + + export CONTEXT="${KUBERNETES_PROVIDER}" + export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG} + # KUBECONFIG determines the file we write to, but it may not exist yet + if [[ ! -e "${KUBECONFIG}" ]]; then + mkdir -p $(dirname "${KUBECONFIG}") + touch "${KUBECONFIG}" + fi + + local token="$(cut -d, -f1 ${provider_root}/certs/apiserver/token-users)" + "${kubectl}" config set-cluster "${CONTEXT}" --server="${KUBE_SERVER}" --certificate-authority="${provider_root}/certs/root-ca.crt" + "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="cluster-admin" + "${kubectl}" config set-credentials cluster-admin --token="${token}" + "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}" + + echo "Wrote config for ${CONTEXT} to ${KUBECONFIG}" 1>&2 +} + +# Perform preparations required to run e2e tests +function prepare-e2e { + echo "TODO: prepare-e2e" 1>&2 +} + +# Execute prior to running tests to build a release if required for env +function test-build-release { + # Make a release + export KUBERNETES_CONTRIB=mesos + export KUBE_RELEASE_RUN_TESTS=N + "${KUBE_ROOT}/build/release.sh" +} + +# Must ensure that the following ENV vars are set +function detect-master { + # echo "KUBE_MASTER: $KUBE_MASTER" 1>&2 + + docker_id=$(docker ps --filter="name=docker_apiserver" --quiet) + if [[ "${docker_id}" == *'\n'* ]]; then + echo "ERROR: Multiple API Servers running in docker" 1>&2 + return 1 + fi + + master_ip=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "${docker_id}") + master_port=6443 + + KUBE_MASTER_IP="${master_ip}:${master_port}" + KUBE_SERVER="https://${KUBE_MASTER_IP}" + + echo "KUBE_MASTER_IP: $KUBE_MASTER_IP" 1>&2 +} + +# Get minion IP addresses and store in KUBE_MINION_IP_ADDRESSES[] +# These Mesos slaves MAY host Kublets, +# but might not have a Kublet running unless a kubernetes task has been scheduled on them. +function detect-minions { + docker_ids=$(docker ps --filter="name=docker_mesosslave" --quiet) + while read -r docker_id; do + minion_ip=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "${docker_id}") + KUBE_MINION_IP_ADDRESSES+=("${minion_ip}") + done <<< "$docker_ids" + echo "KUBE_MINION_IP_ADDRESSES: [${KUBE_MINION_IP_ADDRESSES[*]}]" 1>&2 +} + +# Verify prereqs on host machine +function verify-prereqs { + echo "TODO: verify-prereqs" 1>&2 + # Verify that docker, docker-compose, and jq exist + # Verify that all the right docker images exist: + # mesosphere/kubernetes-mesos-test, etc. +} + +# Instantiate a kubernetes cluster. +function kube-up { + # TODO: version images (k8s version, git sha, and dirty state) to avoid re-building them every time. + if [ "${MESOS_DOCKER_SKIP_BUILD:-false}" != "true" ]; then + echo "Building Docker images" 1>&2 + "${provider_root}/km/build.sh" + "${provider_root}/test/build.sh" + fi + + local certdir="${provider_root}/certs" + + # create mount volume dirs + local apiserver_certs_dir="${certdir}/apiserver" + local controller_certs_dir="${certdir}/controller" + # clean certs directory from previous runs + if [ -d "${apiserver_certs_dir}" ]; then + rm -rf "${apiserver_certs_dir}"/* + fi + mkdir -p "${apiserver_certs_dir}" "${controller_certs_dir}" + + echo "Creating root certificate authority" 1>&2 + cluster::mesos::docker::create_root_certificate_authority "${certdir}" + cp "${certdir}/root-ca.crt" "${controller_certs_dir}/" + + echo "Creating service-account rsa key" 1>&2 + cluster::mesos::docker::create_rsa_key "${certdir}/service-accounts.key" + cp "${certdir}/service-accounts.key" "${apiserver_certs_dir}/" + cp "${certdir}/service-accounts.key" "${controller_certs_dir}/" + + echo "Creating cluster-admin token user" 1>&2 + cluster::mesos::docker::create_token_user "cluster-admin" > "${apiserver_certs_dir}/token-users" + + echo "Creating admin basic auth user" 1>&2 + cluster::mesos::docker::create_basic_user "admin" "admin" > "${apiserver_certs_dir}/basic-users" + + echo "Starting ${KUBERNETES_PROVIDER} cluster" 1>&2 + pushd "${provider_root}" > /dev/null + docker-compose up -d + popd > /dev/null + + detect-master + detect-minions + + # The apiserver is waiting for its certificate, which depends on the IP docker chose. + echo "Creating apiserver certificate" 1>&2 + local apiserer_ip="$(cut -f1 -d: <<<${KUBE_MASTER_IP})" + local apiservice_ip="10.10.10.1" + cluster::mesos::docker::create_apiserver_cert \ + "${certdir}" "${apiserver_certs_dir}" "${apiserer_ip}" "${apiservice_ip}" + + # KUBECONFIG needs to exist before run_in_docker mounts it, otherwise it will be owned by root + create-kubeconfig + + # await-health-check could be run locally, but it would require GNU timeout installed on mac... + # "${provider_root}/common/bin/await-health-check" -t=120 ${KUBE_SERVER}/healthz + cluster::mesos::docker::run_in_docker await-health-check -t=120 http://apiserver:8888/healthz + + echo "Deploying Addons" 1>&2 + KUBE_SERVER=${KUBE_SERVER} "${provider_root}/deploy-addons.sh" + + # Wait for addons to deploy + cluster::mesos::docker::await_ready "kube-dns" + cluster::mesos::docker::await_ready "kube-ui" +} + +function validate-cluster { + echo "Validating ${KUBERNETES_PROVIDER} cluster" 1>&2 + + # Do not validate cluster size. There will be zero k8s minions until a pod is created. + # TODO(karlkfi): use componentstatuses or equivelent when it supports non-localhost core components + + # Validate immediate cluster reachability and responsiveness + echo "KubeDNS: $(cluster::mesos::docker::addon_status 'kube-dns')" + echo "KubeUI: $(cluster::mesos::docker::addon_status 'kube-ui')" +} + +# Delete a kubernetes cluster +function kube-down { + echo "Stopping ${KUBERNETES_PROVIDER} cluster" 1>&2 + pushd "${provider_root}" > /dev/null + # Since restoring a stopped cluster is not yet supported, use the nuclear option + docker-compose kill + docker-compose rm -f + popd > /dev/null +} + +function test-setup { + echo "Building required docker images" 1>&2 + "${KUBE_ROOT}/cluster/mesos/docker/km/build.sh" + "${KUBE_ROOT}/cluster/mesos/docker/test/build.sh" + "${KUBE_ROOT}/cluster/mesos/docker/mesos-slave/build.sh" +} + +# Execute after running tests to perform any required clean-up +function test-teardown { + echo "test-teardown" 1>&2 + kube-down +} + +## Below functions used by hack/e2e-suite/services.sh + +# SSH to a node by name or IP ($1) and run a command ($2). +function ssh-to-node { + echo "TODO: ssh-to-node" 1>&2 +} + +# Restart the kube-proxy on a node ($1) +function restart-kube-proxy { + echo "TODO: restart-kube-proxy" 1>&2 +} + +# Restart the apiserver +function restart-apiserver { + echo "TODO: restart-apiserver" 1>&2 +} + +# Waits for a kube-system pod (of the provided name) to have the phase/status "Running". +function cluster::mesos::docker::await_ready { + local pod_name=$1 + local max_attempts=60 + local phase="Unknown" + echo -n "${pod_name}: " + local n=0 + until [ ${n} -ge ${max_attempts} ]; do + phase=$(cluster::mesos::docker::addon_status "${pod_name}") + if [ "${phase}" == "Running" ]; then + break + fi + echo -n "." + n=$[$n+1] + sleep 1 + done + echo "${phase}" + return $([ "${phase}" == "Running" ]; echo $?) +} + +# Prints the status of the kube-system pod specified +function cluster::mesos::docker::addon_status { + local pod_name=$1 + local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + local phase=$("${kubectl}" get pods --namespace=kube-system -l k8s-app=${pod_name} -o template --template="{{(index .items 0).status.phase}}" 2>/dev/null) + phase="${phase:-Unknown}" + echo "${phase}" +} \ No newline at end of file diff --git a/cluster/test-e2e.sh b/cluster/test-e2e.sh new file mode 100755 index 00000000000..f6caae2581a --- /dev/null +++ b/cluster/test-e2e.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Tests a running Kubernetes cluster. +# TODO: move code from hack/ginkgo-e2e.sh to here + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/cluster/kube-env.sh" + +echo "Testing cluster with provider: ${KUBERNETES_PROVIDER}" 1>&2 + +TEST_ARGS="$@" + +echo "Running e2e tests:" 1>&2 +echo "./hack/ginkgo-e2e.sh ${TEST_ARGS}" 1>&2 +exec "${KUBE_ROOT}/hack/ginkgo-e2e.sh" ${TEST_ARGS} diff --git a/docs/getting-started-guides/README.md b/docs/getting-started-guides/README.md index 321457cfb9a..eec916b3968 100644 --- a/docs/getting-started-guides/README.md +++ b/docs/getting-started-guides/README.md @@ -144,7 +144,8 @@ Bare-metal | custom | Fedora | _none_ | [docs](fedora/fedor Bare-metal | custom | Fedora | flannel | [docs](fedora/flannel_multi_node_cluster.md) | | Community ([@aveshagarwal](https://github.com/aveshagarwal)) libvirt | custom | Fedora | flannel | [docs](fedora/flannel_multi_node_cluster.md) | | Community ([@aveshagarwal](https://github.com/aveshagarwal)) KVM | custom | Fedora | flannel | [docs](fedora/flannel_multi_node_cluster.md) | | Community ([@aveshagarwal](https://github.com/aveshagarwal)) -Mesos/GCE | | | | [docs](mesos.md) | | [Community](https://github.com/mesosphere/kubernetes-mesos) ([@jdef](https://github.com/jdef)) +Mesos/Docker | custom | Ubuntu | Docker | [docs](mesos-docker.md) | | Community ([Kubernetes-Mesos Authors](https://github.com/mesosphere/kubernetes-mesos/blob/master/AUTHORS.md)) +Mesos/GCE | | | | [docs](mesos.md) | | Community ([Kubernetes-Mesos Authors](https://github.com/mesosphere/kubernetes-mesos/blob/master/AUTHORS.md)) AWS | CoreOS | CoreOS | flannel | [docs](coreos.md) | | Community GCE | CoreOS | CoreOS | flannel | [docs](coreos.md) | | Community [@pires](https://github.com/pires) Vagrant | CoreOS | CoreOS | flannel | [docs](coreos.md) | | Community ( [@pires](https://github.com/pires), [@AntonioMeireles](https://github.com/AntonioMeireles) ) diff --git a/docs/getting-started-guides/mesos-docker.md b/docs/getting-started-guides/mesos-docker.md new file mode 100644 index 00000000000..1bc5748d6b4 --- /dev/null +++ b/docs/getting-started-guides/mesos-docker.md @@ -0,0 +1,303 @@ + + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + + +The latest 1.0.x release of this document can be found +[here](http://releases.k8s.io/release-1.0/docs/getting-started-guides/mesos-docker.md). + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +## Getting Started With Kubernetes on Mesos on Docker + +The mesos/docker provider uses docker-compose to launch Kubernetes as a Mesos framework, running in docker with its +dependencies (etcd & mesos). + +### Cluster Goals + +- kubernetes development +- pod/service development +- demoing +- fast deployment +- minimal hardware requirements +- minimal configuration +- entry point for exploration +- simplified networking +- fast end-to-end tests +- local deployment + +Non-Goals: +- high availability +- fault tolerance +- remote deployment +- production usage +- monitoring +- long running +- state persistence across restarts + +### Cluster Topology + +The cluster consists of several docker containers linked together by docker-managed hostnames: + +| Component | Hostname | Description | +|-------------------------------|-----------------------------|-----------------------------------------------------------------------------------------| +| docker-grand-ambassador | | Proxy to allow circular hostname linking in docker | +| etcd | etcd | Key/Value store used by Mesos | +| Mesos Master | mesosmaster1 | REST endpoint for interacting with Mesos | +| Mesos Slave (x2) | mesosslave1
mesosslave2 | Mesos agents that offer resources and run framework executors (e.g. Kubernetes Kublets) | +| Kubernetes API Server | apiserver | REST endpoint for interacting with Kubernetes | +| Kubernetes Controller Manager | controller | | +| Kubernetes Scheduler | scheduler | Schedules container deployment by accepting Mesos offers | + +### Prerequisites + +Required: +- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - version control system +- [Docker CLI](https://docs.docker.com/) - container management command line client +- [Docker Engine](https://docs.docker.com/) - container management daemon + - On Mac, use [Boot2Docker](http://boot2docker.io/) or [Docker Machine](https://docs.docker.com/machine/install-machine/) +- [Docker Compose](https://docs.docker.com/compose/install/) - multi-container application orchestration + +Optional: +- [Virtual Box](https://www.virtualbox.org/wiki/Downloads) - x86 hardware virtualizer + - Required by Boot2Docker and Docker Machine + +#### Install on Mac (Homebrew) + +It's possible to install all of the above via [Homebrew](http://brew.sh/) on a Mac. + +Some steps print instructions for configuring or launching. Make sure each is properly set up before continuing to the next step. + +``` +brew install git +brew install caskroom/cask/brew-cask +brew cask install virtualbox +brew install docker +brew install boot2docker +boot2docker init +boot2docker up +brew install docker-compose +``` + +#### Install on Linux + +Most of the above are available via apt and yum, but depending on your distribution, you may have to install via other +means to get the latest versions. + +It is recommended to use Ubuntu, simply because it best supports AUFS, used by docker to mount volumes. Alternate file +systems may not fully support docker-in-docker. + + +#### Boot2Docker Config (Mac) + +If on a mac using boot2docker, the following steps will make the docker IPs (in the virtualbox VM) reachable from the +host machine (mac). + +1. Set the VM's host-only network to "promiscuous mode": + + ``` + boot2docker stop + VBoxManage modifyvm boot2docker-vm --nicpromisc2 allow-all + boot2docker start + ``` + + This allows the VM to accept packets that were sent to a different IP. + + Since the host-only network routes traffic between VMs and the host, other VMs will also be able to access the docker + IPs, if they have the following route. + +1. Route traffic to docker through the boot2docker IP: + + ``` + sudo route -n add -net 172.17.0.0 $(boot2docker ip) + ``` + + Since the boot2docker IP can change when the VM is restarted, this route may need to be updated over time. + To delete the route later: `sudo route delete 172.17.0.0` + + +### Walkthrough + +1. Checkout source + + ``` + git clone https://github.com/GoogleCloudPlatform/kubernetes + cd kubernetes + ``` + + By default, that will get you the bleeding edge of master branch. + You may want a [release branch](https://github.com/GoogleCloudPlatform/kubernetes/releases) instead, + if you have trouble with master. + +1. Build binaries + + ``` + KUBERNETES_CONTRIB=mesos hack/build-go.sh + ``` + + Alternatively, you can use `make`, if make is installed. + + Unless you're on linux, you'll also need to build the binaries for linux/amd64 (for the docker containers): + + ``` + KUBERNETES_CONTRIB=mesos build/run.sh hack/build-go.sh + ``` + + Breakdown: + - `KUBERNETES_CONTRIB=mesos` - enables building of the contrib/mesos binaries + - `build/run.sh` - executes a command in the build container + - `build-go.sh` - builds the Go binaries for the current architecture (linux/amd64 when in a docker container) + +1. [Optional] Build docker images + + The following docker images are built as part of `./cluster/kube-up.sh`, but it may make sense to build them manually + the first time because it may take a while. In the future some of these may be hosted publicly, but you will always + need to at least rebuild the Kubernetes-Mesos image when using locally built binaries. + + Test image includes all the dependencies required for running e2e tests. + + ``` + ./cluster/mesos/docker/test/build.sh + ``` + + Kubernetes-Mesos image includes the compiled linux binaries. + + ``` + ./cluster/mesos/docker/km/build.sh + ``` + +1. [Optional] Configure Mesos resources + + By default, the mesos-slaves are configured to offer a fixed amount of resources (cpus, memory, disk, ports). + If you want to customize these values, update the `MESOS_RESOURCES` environment variables in `./cluster/mesos/docker/docker-compose.yml`. + If you delete the `MESOS_RESOURCES` environment variables, the resource amounts will be auto-detected based on the host resources, which will over-provision by > 2x. + + If the configured resources are not available on the host, you may want to increase the resources available to Docker Engine. + You may have to increase you VM disk, memory, or cpu allocation in VirtualBox, + [Docker Machine](https://docs.docker.com/machine/#oracle-virtualbox), or + [Boot2Docker](https://ryanfb.github.io/etc/2015/01/28/increasing_boot2docker_allocations_on_os_x.html). + +1. Configure provider + + ``` + export KUBERNETES_PROVIDER=mesos/docker + ``` + + This tells cluster scripts to use the code within `cluster/mesos/docker`. + +1. Create cluster + + ``` + ./cluster/kube-up.sh + ``` + + If you manually built all the above docker images, you can skip that step during kube-up: + + ``` + MESOS_DOCKER_SKIP_BUILD=true ./cluster/kube-up.sh + ``` + + After deploying the cluster, `~/.kube/config` will be created or updated to configure kubectl to target the new cluster. + +1. Explore examples + + To learn more about Pods, Volumes, Labels, Services, and Replication Controllers, start with the + [Kubernetes Walkthrough](../user-guide/walkthrough/). + + To skip to a more advanced example, see the [Guestbook Example](../../examples/guestbook/) + +1. Destroy cluster + + ``` + ./cluster/kube-down.sh + ``` + +### Addons + +The `kube-up` for the mesos/docker provider will automatically deploy KubeDNS and KubeUI addons as pods/services. + +Check their status with: + +``` +./cluster/kubectl.sh get pods --namespace=kube-system +``` + +#### KubeUI + +The web-based Kubernetes UI is accessible in a browser through the API Server proxy: `https://:6443/ui/`. + +By default, basic-auth is configured with user `admin` and password `admin`. + +The IP of the API Server can be found using `./cluster/kubectl.sh cluster-info`. + + +### End To End Testing + +Warning: e2e tests can take a long time to run. You may not want to run them immediately if you're just getting started. + +While your cluster is up, you can run the end-to-end tests: + +``` +./cluster/test-e2e.sh +``` + +Notable parameters: +- Increase the logging verbosity: `-v=2` +- Run only a subset of the tests (regex matching): `-ginkgo.focus=` + +To build, deploy, test, and destroy, all in one command (plus unit & integration tests): + +``` +make test_e2e +``` + + +### Kubernetes CLI + +When compiling from source, it's simplest to use the `./cluster/kubectl.sh` script, which detects your platform & +architecture and proxies commands to the appropriate `kubectl` binary. + +ex: `./cluster/kubectl.sh get pods` + + +### Helpful scripts + +- Kill all docker containers + + ``` + docker ps -q -a | xargs docker rm -f + ``` + +- Clean up unused docker volumes + + ``` + docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes + ``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/getting-started-guides/mesos-docker.md?pixel)]() +