Add docker-compose cluster that runs with mesos

cluster/mesos/docker/.gitignore

@@ -0,0 +1 @@
+certs

cluster/mesos/docker/common/bin/await-file

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Wait for a file to exist in the filesystem.
+# Block up to the timeout duration in seconds (default: 10).
+# Usage: await-health-check [-t=<duration>] <address>
+# Requires: timeout, file
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+duration=10
+if [[ "${1:-}" == "-t="* ]]; then
+    duration="${1:3}"
+    [ -z "${duration}" ] && echo "Invalid duration supplied" && exit 1
+    shift
+fi
+
+file="$1"
+[ -z "$file" ] && echo "No file supplied" && exit 1
+
+echo "Waiting (up to ${duration}s) for ${file} to exist"
+if ! timeout "${duration}" bash -c "while [ ! -f \"${file}\" ]; do sleep 0.5; done"; then
+    echo "Timed out"
+    exit 1
+fi
+
+echo "File ${file} exists now!"

cluster/mesos/docker/common/bin/await-health-check

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Wait for a service to accept connections.
+# Block up to the timeout duration in seconds (default: 10).
+# Usage: await-health-check [-t=<duration>] <address>
+# Requires: timeout, health-check
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+duration=10
+if [[ "${1:-}" == "-t="* ]]; then
+    duration="${1:3}"
+    [ -z "${duration}" ] && echo "Invalid duration supplied" && exit 1
+    shift
+fi
+
+address=${1:-}
+[ -z "${address}" ] && echo "No address supplied" && exit 1
+
+bin=$(cd $(dirname $0) && pwd -P)
+
+echo "Waiting (up to ${duration}s) for ${address} to be healthy"
+if ! timeout "${duration}" bash -c "while ! ${bin}/health-check ${address}; do sleep 0.5; done"; then
+    echo "Timed out"
+    exit 1
+fi
+
+echo "Health check of ${address} succeeded!"

cluster/mesos/docker/common/bin/health-check

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Curl an endpoint and expect it to respond with status 200.
+# Usage: health-check <address>
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+address=${1:-}
+[ -z "${address}" ] && echo "No address supplied" && exit 1
+
+status=$(curl -s -o /dev/null -w '%{http_code}' ${address})
+if [[ "${status}" == '200' ]]; then
+    exit 0
+fi
+if [[ "${status}" == '000' ]]; then
+    exit 7
+fi
+
+exit 1

cluster/mesos/docker/common/bin/resolveip

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Resolve an IP from a hostname (using getent)
+# Usage: resolveip <hostname>
+# Requires: getent
+# TODO: Mac support
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+hostname=$1
+[ -z "${hostname}" ] && echo "No hostname supplied" && exit 1
+
+getent hosts "${hostname}" | cut -d' ' -f1 | sort -u | tail -1

cluster/mesos/docker/config-default.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Contains configuration values for interacting with the mesos/docker cluster
+
+NUM_MINIONS=${NUM_MINIONS:-2}
+INSTANCE_PREFIX="${INSTANCE_PREFIX:-kubernetes}"
+MASTER_NAME="${INSTANCE_PREFIX}-master"
+MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
+
+SERVICE_CLUSTER_IP_RANGE=10.10.10.0/24
+
+# Extra options to set on the Docker command line.  This is useful for setting
+# --insecure-registry for local registries.
+DOCKER_OPTS=""
+
+# Optional: Deploy cluster DNS.
+#ENABLE_CLUSTER_DNS=false
+ENABLE_CLUSTER_DNS=true
+DNS_SERVER_IP="10.10.10.10"
+DNS_DOMAIN="cluster.local"
+DNS_REPLICAS=1
+
+# Optional: Deploy cluster web interface.
+ENABLE_CLUSTER_UI=true

cluster/mesos/docker/config-test.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Contains configuration values for interacting with the docker-compose cluster in test mode
+#Set NUM_MINIONS to minimum required for testing.
+NUM_MINIONS=2
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../..
+source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/config-default.sh"

cluster/mesos/docker/deploy-addons.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deploy the addon services after the cluster is available
+# TODO: integrate with or use /cluster/saltbase/salt/kube-addons/kube-addons.sh
+# Requires:
+#   ENABLE_CLUSTER_DNS (Optional) - 'Y' to deploy kube-dns
+#   KUBE_SERVER (Optional) - url to the api server for configuring kube-dns
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd)
+source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/${KUBE_CONFIG_FILE-"config-default.sh"}"
+source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util-temp-dir.sh"
+kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
+
+
+function deploy_dns {
+  echo "Deploying DNS Addon" 1>&2
+  local workspace=$(pwd)
+
+  # Process salt pillar templates manually
+  sed -e "s/{{ pillar\['dns_replicas'\] }}/${DNS_REPLICAS}/g;s/{{ pillar\['dns_domain'\] }}/${DNS_DOMAIN}/g" "${KUBE_ROOT}/cluster/addons/dns/skydns-rc.yaml.in" > "${workspace}/skydns-rc.yaml"
+  sed -e "s/{{ pillar\['dns_server'\] }}/${DNS_SERVER_IP}/g" "${KUBE_ROOT}/cluster/addons/dns/skydns-svc.yaml.in" > "${workspace}/skydns-svc.yaml"
+
+  # Use kubectl to create skydns rc and service
+  "${kubectl}" create -f "${workspace}/skydns-rc.yaml"
+  "${kubectl}" create -f "${workspace}/skydns-svc.yaml"
+}
+
+function deploy_ui {
+  echo "Deploying UI Addon" 1>&2
+
+  # Use kubectl to create ui rc and service
+  "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/kube-ui/kube-ui-rc.yaml"
+  "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/kube-ui/kube-ui-svc.yaml"
+}
+
+# create the kube-system namespace
+"${kubectl}" create -f "${KUBE_ROOT}/cluster/mesos/docker/kube-system-ns.yaml"
+
+if [ "${ENABLE_CLUSTER_DNS}" == true ]; then
+  cluster::mesos::docker::run_in_temp_dir 'k8sm-dns' 'deploy_dns'
+fi
+
+if [ "${ENABLE_CLUSTER_UI}" == true ]; then
+  deploy_ui
+fi

cluster/mesos/docker/docker-compose.yml

@@ -0,0 +1,157 @@
+ambassador:
+  image: cpuguy83/docker-grand-ambassador:0.9.1
+  volumes:
+  - /var/run/docker.sock:/var/run/docker.sock
+  command: "-name docker_apiserver_1"
+etcd:
+  hostname: etcd
+  image: quay.io/coreos/etcd:v2.0.12
+  ports: [ "4001:4001" ]
+  command: >
+    --listen-client-urls 'http://etcd:4001'
+    --advertise-client-urls 'http://etcd:4001'
+    --initial-cluster-state new
+mesosmaster1:
+  hostname: mesosmaster1
+  image: mesosphere/mesos:0.22.0-1.0.ubuntu1404
+  entrypoint: [ "mesos-master" ]
+  ports: [ "5050:5050" ]
+  environment:
+  - MESOS_HOSTNAME=mesosmaster1
+  - MESOS_PORT=5050
+  - MESOS_LOG_DIR=/var/log/mesos
+  - MESOS_QUORUM=1
+  - MESOS_REGISTRY=in_memory
+  - MESOS_WORK_DIR=/var/lib/mesos
+  links:
+  - etcd
+  - "ambassador:apiserver"
+mesosslave1:
+  hostname: mesosslave1
+  privileged: true
+  image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404
+  entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave1 | cut -d' ' -f1 | sort -u | tail -1)" ]
+  command: ~
+  environment:
+  - MESOS_MASTER=mesosmaster1:5050
+  - MESOS_PORT=5051
+  - MESOS_LOG_DIR=/var/log/mesos
+  - MESOS_LOGGING_LEVEL=INFO
+  - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099]
+  - MESOS_SWITCH_USER=0
+  - MESOS_CONTAINERIZERS=docker,mesos
+  - DOCKER_NETWORK_OFFSET=0.0.1.0
+  - DOCKER_DAEMON_ARGS=--log-level=error
+  links:
+  - etcd
+  - mesosmaster1
+  - "ambassador:apiserver"
+  volumes:
+  - /var/tmp/mesosslave1:/var/lib/docker
+mesosslave2:
+  hostname: mesosslave2
+  privileged: true
+  image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404
+  entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave2 | cut -d' ' -f1 | sort -u | tail -1)" ]
+  command: ~
+  environment:
+  - MESOS_MASTER=mesosmaster1:5050
+  - MESOS_PORT=5051
+  - MESOS_LOG_DIR=/var/log/mesos
+  - MESOS_LOGGING_LEVEL=INFO
+  - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099]
+  - MESOS_SWITCH_USER=0
+  - MESOS_CONTAINERIZERS=docker,mesos
+  - DOCKER_NETWORK_OFFSET=0.0.2.0
+  - DOCKER_DAEMON_ARGS=--log-level=error
+  links:
+  - etcd
+  - mesosmaster1
+  - "ambassador:apiserver"
+  volumes:
+  - /var/tmp/mesosslave2:/var/lib/docker
+apiserver:
+  hostname: apiserver
+  image: mesosphere/kubernetes-mesos
+  entrypoint:
+  - /bin/bash
+  - "-c"
+  - >
+    echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" &&
+    (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo "  mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) &&
+    await-health-check http://etcd:4001/health &&
+    await-health-check http://mesosmaster1:5050/health &&
+    await-file -t=60 /var/run/kubernetes/auth/apiserver.crt &&
+    km apiserver
+    --address=$(resolveip apiserver)
+    --external-hostname=apiserver
+    --etcd-servers=http://etcd:4001
+    --port=8888
+    --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+    --authorization-mode=AlwaysAllow
+    --token-auth-file=/var/run/kubernetes/auth/token-users
+    --basic-auth-file=/var/run/kubernetes/auth/basic-users
+    --service-account-key-file=/var/run/kubernetes/auth/service-accounts.key
+    --service-cluster-ip-range=10.10.10.0/24
+    --service-node-port-range=30000-32767
+    --cloud-provider=mesos
+    --cloud-config=/opt/mesos-cloud.conf
+    --tls-cert-file=/var/run/kubernetes/auth/apiserver.crt
+    --tls-private-key-file=/var/run/kubernetes/auth/apiserver.key
+    --v=2
+  ports: [ "8888:8888", "6443:6443" ]
+  volumes:
+  - ./certs/apiserver:/var/run/kubernetes/auth
+  links:
+  - etcd
+  - mesosmaster1
+controller:
+  hostname: controller
+  image: mesosphere/kubernetes-mesos
+  entrypoint:
+  - /bin/bash
+  - "-c"
+  - >
+    echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" &&
+    (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo "  mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) &&
+    await-health-check -t=60 http://mesosmaster1:5050/health &&
+    await-health-check -t=60 http://apiserver:8888/healthz &&
+    km controller-manager
+    --master=http://apiserver:8888
+    --cloud-config=/opt/mesos-cloud.conf
+    --service-account-private-key-file=/var/run/kubernetes/auth/service-accounts.key
+    --root-ca-file=/var/run/kubernetes/auth/root-ca.crt
+    --v=2
+  volumes:
+  - ./certs/controller:/var/run/kubernetes/auth
+  links:
+  - mesosmaster1
+  - apiserver
+scheduler:
+  hostname: scheduler
+  image: mesosphere/kubernetes-mesos
+  entrypoint:
+  - /bin/bash
+  - "-c"
+  - >
+    echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" &&
+    (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo "  mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) &&
+    await-health-check http://etcd:4001/health &&
+    await-health-check http://mesosmaster1:5050/health &&
+    await-health-check -t=60 http://apiserver:8888/healthz &&
+    km scheduler
+    --address=$(resolveip scheduler)
+    --hostname-override=scheduler
+    --etcd-servers=http://etcd:4001
+    --mesos-user=root
+    --api-servers=http://apiserver:8888
+    --mesos-master=mesosmaster1:5050
+    --cluster-dns=10.10.10.10
+    --cluster-domain=cluster.local
+    --v=2
+  links:
+  - etcd
+  - mesosmaster1
+  - mesosslave1
+  - mesosslave2
+  - apiserver

cluster/mesos/docker/km/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:14.04.2
+MAINTAINER Mesosphere <support@mesosphere.io>
+
+RUN locale-gen en_US.UTF-8
+RUN dpkg-reconfigure locales
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
+RUN apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \
+        wget \
+        curl \
+        && \
+    apt-get clean
+
+COPY ./bin/* /usr/local/bin/
+ADD ./opt/mesos-cloud.conf /opt/

cluster/mesos/docker/km/build.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds a docker image that contains the kubernetes-mesos binaries.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+IMAGE_REPO=${IMAGE_REPO:-mesosphere/kubernetes-mesos}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+
+script_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
+KUBE_ROOT=$(cd ${script_dir}/../../../.. && pwd -P)
+
+# Find a platform specific binary, whether it was cross compiled, locally built, or downloaded.
+find-binary() {
+  local lookfor="${1}"
+  local platform="${2}"
+  local locations=(
+    "${KUBE_ROOT}/_output/dockerized/bin/${platform}/${lookfor}"
+    "${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}"
+    "${KUBE_ROOT}/platforms/${platform}/${lookfor}"
+  )
+  local bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
+  echo -n "${bin}"
+}
+
+km_path=$(find-binary km linux/amd64)
+if [ -z "$km_path" ]; then
+  echo "Failed to find km binary" 1>&2
+  exit 1
+fi
+kube_bin_path=$(dirname ${km_path})
+common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P)
+
+cd "${KUBE_ROOT}"
+
+# create temp workspace to place compiled binaries with image-specific scripts
+# create temp workspace dir in KUBE_ROOT to avoid permission issues of TMPDIR on mac os x
+workspace=$(env TMPDIR=$PWD mktemp -d -t "k8sm-workspace-XXXXXX")
+echo "Workspace created: ${workspace}"
+
+cleanup() {
+  rm -rf "${workspace}"
+  echo "Workspace deleted: ${workspace}"
+}
+trap 'cleanup' EXIT
+
+# setup workspace to mirror script dir (dockerfile expects /bin & /opt)
+echo "Copying files to workspace"
+
+# binaries & scripts
+mkdir -p "${workspace}/bin"
+#cp "${script_dir}/bin/"* "${workspace}/bin/"
+cp "${common_bin_path}/"* "${workspace}/bin/"
+cp "${kube_bin_path}/"* "${workspace}/bin/"
+
+# config
+mkdir -p "${workspace}/opt"
+cp "${script_dir}/opt/"* "${workspace}/opt/"
+
+# docker
+cp "${script_dir}/Dockerfile" "${workspace}/"
+
+cd "${workspace}"
+
+# build docker image
+echo "Building docker image ${IMAGE_REPO}:${IMAGE_TAG}"
+set -o xtrace
+docker build -t ${IMAGE_REPO}:${IMAGE_TAG} "$@" .
+set +o xtrace
+echo "Built docker image ${IMAGE_REPO}:${IMAGE_TAG}"

cluster/mesos/docker/km/opt/mesos-cloud.conf

@@ -0,0 +1,3 @@
+[mesos-cloud]
+  http-client-timeout = 5s
+  state-cache-ttl     = 20s

cluster/mesos/docker/kube-system-ns.yaml

@@ -0,0 +1,6 @@
+kind: "Namespace"
+apiVersion: "v1"
+metadata:
+  name: "kube-system"
+  labels:
+    name: "kube-system"

cluster/mesos/docker/test/Dockerfile

@@ -0,0 +1,49 @@
+FROM golang:1.4.2
+MAINTAINER Mesosphere <support@mesosphere.io>
+
+# docker.io is suppossed to be in backports, but it's not there yet.
+# https://github.com/docker/docker/issues/13253
+# http://docs.docker.com/installation/debian/#debian-jessie-80-64-bit
+#RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" >> /etc/apt/sources.list
+#RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list
+
+RUN apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \
+        wget \
+        curl \
+        g++ \
+        make \
+        mercurial \
+        git \
+        rsync \
+        patch \
+        python \
+        python-pip \
+        apt-transport-https \
+        && \
+    apt-get clean
+
+# Install latest Docker
+# RUN curl -sSL https://get.docker.com/ubuntu/ | sh
+
+# Install Docker 1.6.2 (docker 1.7 has regressions)
+RUN echo deb https://get.docker.com/ubuntu docker main > /etc/apt/sources.list.d/docker.list && \
+    apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \
+    apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \
+        lxc-docker-1.6.2 \
+        && \
+    apt-get clean
+
+RUN pip install -U docker-compose
+
+RUN go get github.com/tools/godep
+
+RUN mkdir -p /go/src/github.com/GoogleCloudPlatform/kubernetes
+WORKDIR /go/src/github.com/GoogleCloudPlatform/kubernetes
+
+COPY ./bin/* /usr/local/bin/
+
+RUN install-etcd.sh
+
+ENTRYPOINT [ "bash" ]

cluster/mesos/docker/test/bin/install-etcd.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Installs etcd into /usr/local/bin
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ETCD_VERSION=${ETCD_VERSION:-v2.0.11}
+
+full_name=etcd-${ETCD_VERSION}-linux-amd64
+archive_url=https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/${full_name}.tar.gz
+
+download_dir=/tmp/etcd-${ETCD_VERSION}
+
+mkdir ${download_dir}
+
+function cleanup {
+    rm -rf ${download_dir}
+}
+
+trap cleanup EXIT
+
+cd ${download_dir}
+
+echo "Downloading etcd (${archive_url})..."
+curl -s -L ${archive_url} | tar xvz
+
+echo "Installing etcd (/usr/local/bin/etcd)..."
+mv ./${full_name}/etcd* /usr/local/bin/

cluster/mesos/docker/test/build.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+IMAGE_REPO=${IMAGE_REPO:-mesosphere/kubernetes-mesos-test}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+
+script_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
+common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P)
+KUBE_ROOT=$(cd ${script_dir}/../../../.. && pwd -P)
+
+cd "${KUBE_ROOT}"
+
+# create temp workspace to place common scripts with image-specific scripts
+# create temp workspace dir in KUBE_ROOT to avoid permission issues of TMPDIR on mac os x
+workspace=$(env TMPDIR=$PWD mktemp -d -t "k8sm-test-workspace-XXXXXX")
+echo "Workspace created: ${workspace}"
+
+cleanup() {
+  rm -rf "${workspace}"
+  echo "Workspace deleted: ${workspace}"
+}
+trap 'cleanup' EXIT
+
+# setup workspace to mirror script dir (dockerfile expects /bin)
+set -x
+echo "Copying files to workspace"
+
+# binaries & scripts
+mkdir -p "${workspace}/bin"
+cp -a "${common_bin_path}/"* "${workspace}/bin/"
+cp -a "${script_dir}/bin/"* "${workspace}/bin/"
+
+# docker
+cp -a "${script_dir}/Dockerfile" "${workspace}/"
+
+cd "${workspace}"
+
+echo "Building docker image ${IMAGE_REPO}:${IMAGE_TAG}"
+set -o xtrace
+docker build -t ${IMAGE_REPO}:${IMAGE_TAG} "$@" .
+set +o xtrace
+echo "Built docker image ${IMAGE_REPO}:${IMAGE_TAG}"

cluster/mesos/docker/util-ssl.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sourcable SSL functions
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir=$(cd "$(dirname ${BASH_SOURCE})" && pwd -P)
+source "${script_dir}/util-temp-dir.sh"
+
+function cluster::mesos::docker::find_openssl_config {
+  for candidate in "/etc/ssl/openssl.cnf" "/System/Library/OpenSSL/openssl.cnf"; do
+    if [ -f "${candidate}" ]; then
+      echo "${candidate}"
+      return 0
+    fi
+  done
+  echo "ERROR: cannot find openssl.cnf" 1>&2
+  return 1
+}
+
+function cluster::mesos::docker::create_root_certificate_authority {
+  local certdir="$1"
+
+  openssl req -nodes -newkey rsa:2048 \
+    -keyout "${certdir}/root-ca.key" \
+    -out "${certdir}/root-ca.csr" \
+    -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com"
+
+  openssl x509 -req -days 3650 \
+    -in "${certdir}/root-ca.csr" \
+    -out "${certdir}/root-ca.crt" \
+    -signkey "${certdir}/root-ca.key"
+}
+
+# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names.
+# Uses the current dir for scratch work.
+function cluster::mesos::docker::create_apiserver_cert_inner {
+  local in_dir="$1" # must contain root-ca.crt & root-ca.key
+  local out_dir="$2"
+  local apiserver_ip="$3"
+  local service_ip="$4"
+  local workspace="$(pwd)"
+
+  mkdir -p "${out_dir}"
+
+  local OPENSSL_CNF=$(cluster::mesos::docker::find_openssl_config)
+
+  # create apiserver key and certificate sign request
+  local SANS="IP:${apiserver_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local"
+  openssl req -nodes -newkey rsa:2048 \
+    -keyout "${workspace}/apiserver.key" -out "${workspace}/apiserver.csr" \
+    -reqexts SAN -config <(cat "${OPENSSL_CNF}"; echo -e "[SAN]\nsubjectAltName=$SANS") \
+    -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com"
+
+  # sign with root-ca
+  mkdir -p ${workspace}/demoCA/newcerts
+  touch ${workspace}/demoCA/index.txt
+  echo 1000 > ${workspace}/demoCA/serial
+  openssl ca -cert "${in_dir}/root-ca.crt" -keyfile "${in_dir}/root-ca.key" \
+    -batch -days 3650 -in "${workspace}/apiserver.csr" \
+    -config <(sed 's/.*\(copy_extensions = copy\)/\1/' ${OPENSSL_CNF}) >/dev/null
+
+  # check certificate for subjectAltName extension
+  if ! openssl x509 -in "${workspace}/demoCA/newcerts/1000.pem" -text -noout | grep -q kubernetes.default.svc.cluster.local; then
+    echo "ERROR: openssl failed to add subjectAltName extension" 1>&2
+    return 1
+  fi
+
+  # write to out_dir
+  cp "${workspace}/demoCA/newcerts/1000.pem" "${out_dir}/apiserver.crt"
+  cp "${workspace}/apiserver.key" "${out_dir}/"
+}
+
+# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names.
+function cluster::mesos::docker::create_apiserver_cert {
+  local in_dir="$1" # must contain root-ca.crt & root-ca.key
+  local out_dir="$2"
+  local apiserver_ip="$3"
+  local service_ip="$4"
+
+  cluster::mesos::docker::run_in_temp_dir "k8sm-certs" \
+    "cluster::mesos::docker::create_apiserver_cert_inner" \
+      "${in_dir}" "${out_dir}" "${apiserver_ip}" "${service_ip}"
+}
+
+# Creates an rsa key (for signing service accounts).
+function cluster::mesos::docker::create_rsa_key {
+  local key_file_path="$1"
+  openssl genrsa -out "${key_file_path}" 2048
+}
+
+# Creates a k8s token auth user file.
+# See /docs/admin/authentication.md
+function cluster::mesos::docker::create_token_user {
+  local user_name="$1"
+  echo "$(openssl rand -hex 32),${user_name},${user_name}"
+}
+
+# Creates a k8s basic auth user file.
+# See /docs/admin/authentication.md
+function cluster::mesos::docker::create_basic_user {
+  local user_name="$1"
+  local password="$2"
+  echo "${password},${user_name},${user_name}"
+}

cluster/mesos/docker/util-temp-dir.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sourcable temp directory functions
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Runs the supplied command string in a temporary workspace directory.
+function cluster::mesos::docker::run_in_temp_dir {
+  prefix="$1"
+  shift
+  cmd="$@"
+
+  # create temp WORKSPACE dir in current dir to avoid permission issues of TMPDIR on mac os x
+  local -r workspace=$(env TMPDIR=$(pwd) mktemp -d -t "${prefix}-XXXXXX")
+  echo "Workspace created: ${workspace}" 1>&2
+
+  cleanup() {
+    rm -rf "${workspace}"
+    echo "Workspace deleted: ${workspace}" 1>&2
+  }
+  trap 'cleanup' EXIT
+
+  pushd "${workspace}" > /dev/null
+  (${cmd}) || return $?
+  popd > /dev/null
+
+  trap - EXIT
+  cleanup
+}

cluster/mesos/docker/util.sh

@@ -0,0 +1,295 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Example:
+# export KUBERNETES_PROVIDER=mesos/docker
+# go run hack/e2e.go -v -up -check_cluster_size=false
+# go run hack/e2e.go -v -test -check_version_skew=false
+# go run hack/e2e.go -v -down
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd)
+provider_root="${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}"
+
+source "${provider_root}/${KUBE_CONFIG_FILE-"config-default.sh"}"
+source "${KUBE_ROOT}/cluster/common.sh"
+source "${provider_root}/util-ssl.sh"
+
+
+# Run kubernetes scripts inside docker.
+# This bypasses the need to set up network routing when running docker in a VM (e.g. boot2docker).
+# Trap signals and kills the docker container for better signal handing
+function cluster::mesos::docker::run_in_docker {
+  entrypoint="$1"
+  if [[ "${entrypoint}" = "./"* ]]; then
+    # relative to project root
+    entrypoint="/go/src/github.com/GoogleCloudPlatform/kubernetes/${entrypoint}"
+  fi
+  shift
+  args="$@"
+
+  container_id=$(
+    docker run \
+      -d \
+      -e "KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER}" \
+      -v "${KUBE_ROOT}:/go/src/github.com/GoogleCloudPlatform/kubernetes" \
+      -v "$(dirname ${KUBECONFIG}):/root/.kube" \
+      -v "/var/run/docker.sock:/var/run/docker.sock" \
+      --link docker_mesosmaster1_1:mesosmaster1 \
+      --link docker_mesosslave1_1:mesosslave1 \
+      --link docker_mesosslave2_1:mesosslave2 \
+      --link docker_apiserver_1:apiserver \
+      --entrypoint="${entrypoint}" \
+      mesosphere/kubernetes-mesos-test \
+      ${args}
+  )
+
+  docker logs -f "${container_id}" &
+
+  # trap and kill for better signal handing
+  trap 'echo "Killing container ${container_id}" 1>&2 && docker kill ${container_id}' INT TERM
+  exit_status=$(docker wait "${container_id}")
+  trap - INT TERM
+
+  if [ "$exit_status" != 0 ]; then
+    echo "Exited ${exit_status}" 1>&2
+  fi
+
+  docker rm -f "${container_id}" > /dev/null
+
+  return "${exit_status}"
+}
+
+# Generate kubeconfig data for the created cluster.
+function create-kubeconfig {
+  local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
+
+  export CONTEXT="${KUBERNETES_PROVIDER}"
+  export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
+  # KUBECONFIG determines the file we write to, but it may not exist yet
+  if [[ ! -e "${KUBECONFIG}" ]]; then
+    mkdir -p $(dirname "${KUBECONFIG}")
+    touch "${KUBECONFIG}"
+  fi
+
+  local token="$(cut -d, -f1 ${provider_root}/certs/apiserver/token-users)"
+  "${kubectl}" config set-cluster "${CONTEXT}" --server="${KUBE_SERVER}" --certificate-authority="${provider_root}/certs/root-ca.crt"
+  "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="cluster-admin"
+  "${kubectl}" config set-credentials cluster-admin --token="${token}"
+  "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}"
+
+   echo "Wrote config for ${CONTEXT} to ${KUBECONFIG}" 1>&2
+}
+
+# Perform preparations required to run e2e tests
+function prepare-e2e {
+  echo "TODO: prepare-e2e" 1>&2
+}
+
+# Execute prior to running tests to build a release if required for env
+function test-build-release {
+  # Make a release
+  export KUBERNETES_CONTRIB=mesos
+  export KUBE_RELEASE_RUN_TESTS=N
+  "${KUBE_ROOT}/build/release.sh"
+}
+
+# Must ensure that the following ENV vars are set
+function detect-master {
+  #  echo "KUBE_MASTER: $KUBE_MASTER" 1>&2
+
+  docker_id=$(docker ps --filter="name=docker_apiserver" --quiet)
+  if [[ "${docker_id}" == *'\n'* ]]; then
+    echo "ERROR: Multiple API Servers running in docker" 1>&2
+    return 1
+  fi
+
+  master_ip=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "${docker_id}")
+  master_port=6443
+
+  KUBE_MASTER_IP="${master_ip}:${master_port}"
+  KUBE_SERVER="https://${KUBE_MASTER_IP}"
+
+  echo "KUBE_MASTER_IP: $KUBE_MASTER_IP" 1>&2
+}
+
+# Get minion IP addresses and store in KUBE_MINION_IP_ADDRESSES[]
+# These Mesos slaves MAY host Kublets,
+# but might not have a Kublet running unless a kubernetes task has been scheduled on them.
+function detect-minions {
+  docker_ids=$(docker ps --filter="name=docker_mesosslave" --quiet)
+  while read -r docker_id; do
+    minion_ip=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "${docker_id}")
+    KUBE_MINION_IP_ADDRESSES+=("${minion_ip}")
+  done <<< "$docker_ids"
+  echo "KUBE_MINION_IP_ADDRESSES: [${KUBE_MINION_IP_ADDRESSES[*]}]" 1>&2
+}
+
+# Verify prereqs on host machine
+function verify-prereqs {
+  echo "TODO: verify-prereqs" 1>&2
+  # Verify that docker, docker-compose, and jq exist
+  # Verify that all the right docker images exist:
+  # mesosphere/kubernetes-mesos-test, etc.
+}
+
+# Instantiate a kubernetes cluster.
+function kube-up {
+  # TODO: version images (k8s version, git sha, and dirty state) to avoid re-building them every time.
+  if [ "${MESOS_DOCKER_SKIP_BUILD:-false}" != "true" ]; then
+    echo "Building Docker images" 1>&2
+    "${provider_root}/km/build.sh"
+    "${provider_root}/test/build.sh"
+  fi
+
+  local certdir="${provider_root}/certs"
+
+  # create mount volume dirs
+  local apiserver_certs_dir="${certdir}/apiserver"
+  local controller_certs_dir="${certdir}/controller"
+  # clean certs directory from previous runs
+  if [ -d "${apiserver_certs_dir}" ]; then
+    rm -rf "${apiserver_certs_dir}"/*
+  fi
+  mkdir -p "${apiserver_certs_dir}" "${controller_certs_dir}"
+
+  echo "Creating root certificate authority" 1>&2
+  cluster::mesos::docker::create_root_certificate_authority "${certdir}"
+  cp "${certdir}/root-ca.crt" "${controller_certs_dir}/"
+
+  echo "Creating service-account rsa key" 1>&2
+  cluster::mesos::docker::create_rsa_key "${certdir}/service-accounts.key"
+  cp "${certdir}/service-accounts.key" "${apiserver_certs_dir}/"
+  cp "${certdir}/service-accounts.key" "${controller_certs_dir}/"
+
+  echo "Creating cluster-admin token user" 1>&2
+  cluster::mesos::docker::create_token_user "cluster-admin" > "${apiserver_certs_dir}/token-users"
+
+  echo "Creating admin basic auth user" 1>&2
+  cluster::mesos::docker::create_basic_user "admin" "admin" > "${apiserver_certs_dir}/basic-users"
+
+  echo "Starting ${KUBERNETES_PROVIDER} cluster" 1>&2
+  pushd "${provider_root}" > /dev/null
+  docker-compose up -d
+  popd > /dev/null
+
+  detect-master
+  detect-minions
+
+  # The apiserver is waiting for its certificate, which depends on the IP docker chose.
+  echo "Creating apiserver certificate" 1>&2
+  local apiserer_ip="$(cut -f1 -d: <<<${KUBE_MASTER_IP})"
+  local apiservice_ip="10.10.10.1"
+  cluster::mesos::docker::create_apiserver_cert \
+    "${certdir}" "${apiserver_certs_dir}" "${apiserer_ip}" "${apiservice_ip}"
+
+  # KUBECONFIG needs to exist before run_in_docker mounts it, otherwise it will be owned by root
+  create-kubeconfig
+
+  # await-health-check could be run locally, but it would require GNU timeout installed on mac...
+  # "${provider_root}/common/bin/await-health-check" -t=120 ${KUBE_SERVER}/healthz
+  cluster::mesos::docker::run_in_docker await-health-check -t=120 http://apiserver:8888/healthz
+
+  echo "Deploying Addons" 1>&2
+  KUBE_SERVER=${KUBE_SERVER} "${provider_root}/deploy-addons.sh"
+
+  # Wait for addons to deploy
+  cluster::mesos::docker::await_ready "kube-dns"
+  cluster::mesos::docker::await_ready "kube-ui"
+}
+
+function validate-cluster {
+  echo "Validating ${KUBERNETES_PROVIDER} cluster" 1>&2
+
+  # Do not validate cluster size. There will be zero k8s minions until a pod is created.
+  # TODO(karlkfi): use componentstatuses or equivelent when it supports non-localhost core components
+
+  # Validate immediate cluster reachability and responsiveness
+  echo "KubeDNS: $(cluster::mesos::docker::addon_status 'kube-dns')"
+  echo "KubeUI: $(cluster::mesos::docker::addon_status 'kube-ui')"
+}
+
+# Delete a kubernetes cluster
+function kube-down {
+  echo "Stopping ${KUBERNETES_PROVIDER} cluster" 1>&2
+  pushd "${provider_root}" > /dev/null
+  # Since restoring a stopped cluster is not yet supported, use the nuclear option
+  docker-compose kill
+  docker-compose rm -f
+  popd > /dev/null
+}
+
+function test-setup {
+  echo "Building required docker images" 1>&2
+  "${KUBE_ROOT}/cluster/mesos/docker/km/build.sh"
+  "${KUBE_ROOT}/cluster/mesos/docker/test/build.sh"
+  "${KUBE_ROOT}/cluster/mesos/docker/mesos-slave/build.sh"
+}
+
+# Execute after running tests to perform any required clean-up
+function test-teardown {
+  echo "test-teardown" 1>&2
+  kube-down
+}
+
+## Below functions used by hack/e2e-suite/services.sh
+
+# SSH to a node by name or IP ($1) and run a command ($2).
+function ssh-to-node {
+  echo "TODO: ssh-to-node" 1>&2
+}
+
+# Restart the kube-proxy on a node ($1)
+function restart-kube-proxy {
+  echo "TODO: restart-kube-proxy" 1>&2
+}
+
+# Restart the apiserver
+function restart-apiserver {
+  echo "TODO: restart-apiserver" 1>&2
+}
+
+# Waits for a kube-system pod (of the provided name) to have the phase/status "Running".
+function cluster::mesos::docker::await_ready {
+  local pod_name=$1
+  local max_attempts=60
+  local phase="Unknown"
+  echo -n "${pod_name}: "
+  local n=0
+  until [ ${n} -ge ${max_attempts} ]; do
+    phase=$(cluster::mesos::docker::addon_status "${pod_name}")
+    if [ "${phase}" == "Running" ]; then
+      break
+    fi
+    echo -n "."
+    n=$[$n+1]
+    sleep 1
+  done
+  echo "${phase}"
+  return $([ "${phase}" == "Running" ]; echo $?)
+}
+
+# Prints the status of the kube-system pod specified
+function cluster::mesos::docker::addon_status {
+  local pod_name=$1
+  local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
+  local phase=$("${kubectl}" get pods --namespace=kube-system -l k8s-app=${pod_name} -o template --template="{{(index .items 0).status.phase}}" 2>/dev/null)
+  phase="${phase:-Unknown}"
+  echo "${phase}"
+}