diff --git a/cluster/juju/bundles/local.yaml b/cluster/juju/bundles/local.yaml
index 6b1e8bf2b49..cf87a8cd3b8 100644
--- a/cluster/juju/bundles/local.yaml
+++ b/cluster/juju/bundles/local.yaml
@@ -1,53 +1,18 @@
-kubernetes-local:
- services:
- kubernetes-master:
- charm: local:trusty/kubernetes-master
- annotations:
- "gui-x": "600"
- "gui-y": "0"
- expose: true
- options:
- version: "local"
- docker:
- charm: cs:trusty/docker
- num_units: 2
- options:
- latest: true
- annotations:
- "gui-x": "0"
- "gui-y": "0"
- flannel-docker:
- charm: cs:~kubernetes/trusty/flannel-docker
- annotations:
- "gui-x": "0"
- "gui-y": "300"
- kubernetes:
- charm: local:trusty/kubernetes
- annotations:
- "gui-x": "300"
- "gui-y": "300"
- etcd:
- charm: cs:~kubernetes/trusty/etcd
- annotations:
- "gui-x": "300"
- "gui-y": "0"
- relations:
- - - "flannel-docker:network"
- - "docker:network"
- - - "flannel-docker:network"
- - "kubernetes-master:network"
- - - "flannel-docker:docker-host"
- - "docker:juju-info"
- - - "flannel-docker:docker-host"
- - "kubernetes-master:juju-info"
- - - "flannel-docker:db"
- - "etcd:client"
- - - "kubernetes:docker-host"
- - "docker:juju-info"
- - - "etcd:client"
- - "kubernetes:etcd"
- - - "etcd:client"
- - "kubernetes-master:etcd"
- - - "kubernetes-master:minions-api"
- - "kubernetes:api"
- series: trusty
+services:
+ kubernetes:
+ charm: local:trusty/kubernetes
+ annotations:
+ "gui-x": "600"
+ "gui-y": "0"
+ expose: true
+ num_units: 2
+ etcd:
+ charm: cs:~lazypower/trusty/etcd
+ annotations:
+ "gui-x": "300"
+ "gui-y": "0"
+ num_units: 1
+relations:
+ - - "kubernetes:etcd"
+ - "etcd:db"
+series: trusty
diff --git a/cluster/juju/identify-leaders.py b/cluster/juju/identify-leaders.py
new file mode 100755
index 00000000000..01bdec24ff8
--- /dev/null
+++ b/cluster/juju/identify-leaders.py
@@ -0,0 +1,15 @@
+#!/usr/bin/python
+
+from subprocess import check_output
+import yaml
+
+out = check_output(['juju', 'status', 'kubernetes', '--format=yaml'])
+try:
+ parsed_output = yaml.safe_load(out)
+ model = parsed_output['services']['kubernetes']['units']
+ for unit in model:
+ if 'workload-status' in model[unit].keys():
+ if 'leader' in model[unit]['workload-status']['message']:
+ print(unit)
+except:
+ pass
diff --git a/cluster/juju/layers/kubernetes/README.md b/cluster/juju/layers/kubernetes/README.md
new file mode 100644
index 00000000000..cbbdc5bea83
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/README.md
@@ -0,0 +1,74 @@
+# kubernetes
+
+[Kubernetes](https://github.com/kubernetes/kubernetes) is an open
+source system for managing application containers across multiple hosts.
+This version of Kubernetes uses [Docker](http://www.docker.io/) to package,
+instantiate and run containerized applications.
+
+This charm is an encapsulation of the
+[Running Kubernetes locally via
+Docker](https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/docker.md)
+document. The released hyperkube image (`gcr.io/google_containers/hyperkube`)
+is currently pulled from a [Google owned container repository
+repository](https://cloud.google.com/container-registry/). For this charm to
+work it will need access to the repository to `docker pull` the images.
+
+This charm was built from other charm layers using the reactive framework. The
+`layer:docker` is the base layer. For more information please read [Getting
+Started Developing charms](https://jujucharms.com/docs/devel/developer-getting-started)
+
+# Deployment
+The kubernetes charms require a relation to a distributed key value store
+(ETCD) which Kubernetes uses for persistent storage of all of its REST API
+objects.
+
+```
+juju deploy trusty/etcd
+juju deploy local:trusty/kubernetes
+juju add-relation kubernetes etcd
+```
+
+# Configuration
+For your convenience this charm supports some configuration options to set up
+a Kuberentes cluster that works in your environment:
+
+**version**: Set the version of the Kubernetes containers to deploy.
+The default value is "v1.0.6". Changing the version causes the all the
+Kubernetes containers to be restarted.
+
+**cidr**: Set the IP range for the Kubernetes cluster. eg: 10.1.0.0/16
+
+
+## State Events
+While this charm is meant to be a top layer, it can be used to build other
+solutions. This charm sets or removes states from the reactive framework that
+other layers could react appropriately. The states that other layers would be
+interested in are as follows:
+
+**kubelet.available** - The hyperkube container has been run with the kubelet
+service and configuration that started the apiserver, controller-manager and
+scheduler containers.
+
+**proxy.available** - The hyperkube container has been run with the proxy
+service and configuration that handles Kubernetes networking.
+
+**kubectl.package.created** - Indicates the availability of the `kubectl`
+application along with the configuration needed to contact the cluster
+securely. You will need to download the `/home/ubuntu/kubectl_package.tar.gz`
+from the kubernetes leader unit to your machine so you can control the cluster.
+
+**skydns.available** - Indicates when the Domain Name System (DNS) for the
+cluster is operational.
+
+
+# Kubernetes information
+
+ - [Kubernetes github project](https://github.com/kubernetes/kubernetes)
+ - [Kubernetes issue tracker](https://github.com/kubernetes/kubernetes/issues)
+ - [Kubernetes Documenation](https://github.com/kubernetes/kubernetes/tree/master/docs)
+ - [Kubernetes releases](https://github.com/kubernetes/kubernetes/releases)
+
+# Contact
+
+ * Charm Author: Matthew Bruzek <Matthew.Bruzek@canonical.com>
+ * Charm Contributor: Charles Butler <Charles.Butler@canonical.com>
diff --git a/cluster/juju/layers/kubernetes/actions.yaml b/cluster/juju/layers/kubernetes/actions.yaml
new file mode 100644
index 00000000000..82cfd5ac646
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/actions.yaml
@@ -0,0 +1,2 @@
+guestbook-example:
+ description: Launch the guestbook example in your k8s cluster
diff --git a/cluster/juju/layers/kubernetes/actions/guestbook-example b/cluster/juju/layers/kubernetes/actions/guestbook-example
new file mode 100755
index 00000000000..49db127e7d1
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/actions/guestbook-example
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Launch the Guestbook example in Kubernetes. This will use the pod and service
+# definitions from `files/guestbook-example/*.yaml` to launch a leader/follower
+# redis cluster, with a web-front end to collect user data and store in redis.
+# This example app can easily scale across multiple nodes, and exercises the
+# networking, pod creation/scale, service definition, and replica controller of
+# kubernetes.
+#
+# Lifted from github.com/kubernetes/kubernetes/examples/guestbook-example
+
+kubectl create -f files/guestbook-example/redis-master-service.yaml
+kubectl create -f files/guestbook-example/frontend-service.yaml
+
+kubectl create -f files/guestbook-example/frontend-controller.yaml
+
+kubectl create -f files/guestbook-example/redis-master-controller.yaml
+kubectl create -f files/guestbook-example/redis-master-controller.yaml
+
+kubectl create -f files/guestbook-example/redis-slave-service.yaml
+kubectl create -f files/guestbook-example/redis-slave-controller.yaml
diff --git a/cluster/juju/layers/kubernetes/config.yaml b/cluster/juju/layers/kubernetes/config.yaml
new file mode 100644
index 00000000000..1c221b124ed
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/config.yaml
@@ -0,0 +1,14 @@
+options:
+ version:
+ type: string
+ default: "v1.1.7"
+ description: |
+ The version of Kubernetes to use in this charm. The version is
+ inserted in the configuration files that specify the hyperkube
+ container to use when starting a Kubernetes cluster. Changing this
+ value will restart the Kubernetes cluster.
+ cidr:
+ type: string
+ default: 10.1.0.0/16
+ description: |
+ Network CIDR to assign to K8s service groups
diff --git a/cluster/juju/layers/kubernetes/files/guestbook-example/frontend-controller.yaml b/cluster/juju/layers/kubernetes/files/guestbook-example/frontend-controller.yaml
new file mode 100644
index 00000000000..1a48f95b346
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/files/guestbook-example/frontend-controller.yaml
@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: frontend
+ labels:
+ name: frontend
+spec:
+ replicas: 3
+ selector:
+ name: frontend
+ template:
+ metadata:
+ labels:
+ name: frontend
+ spec:
+ containers:
+ - name: php-redis
+ image: gcr.io/google_samples/gb-frontend:v3
+ env:
+ - name: GET_HOSTS_FROM
+ value: dns
+ # If your cluster config does not include a dns service, then to
+ # instead access environment variables to find service host
+ # info, comment out the 'value: dns' line above, and uncomment the
+ # line below.
+ # value: env
+ ports:
+ - containerPort: 80
diff --git a/cluster/juju/layers/kubernetes/files/guestbook-example/frontend-service.yaml b/cluster/juju/layers/kubernetes/files/guestbook-example/frontend-service.yaml
new file mode 100644
index 00000000000..22f0273b095
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/files/guestbook-example/frontend-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: frontend
+ labels:
+ name: frontend
+spec:
+ # if your cluster supports it, uncomment the following to automatically create
+ # an external load-balanced IP for the frontend service.
+ # type: LoadBalancer
+ ports:
+ # the port that this service should serve on
+ - port: 80
+ selector:
+ name: frontend
diff --git a/cluster/juju/layers/kubernetes/files/guestbook-example/redis-master-controller.yaml b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-master-controller.yaml
new file mode 100644
index 00000000000..2eebf2625c3
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-master-controller.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: redis-master
+ labels:
+ name: redis-master
+spec:
+ replicas: 1
+ selector:
+ name: redis-master
+ template:
+ metadata:
+ labels:
+ name: redis-master
+ spec:
+ containers:
+ - name: master
+ image: redis
+ ports:
+ - containerPort: 6379
diff --git a/cluster/juju/layers/kubernetes/files/guestbook-example/redis-master-service.yaml b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-master-service.yaml
new file mode 100644
index 00000000000..b200cd6ec01
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-master-service.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: redis-master
+ labels:
+ name: redis-master
+spec:
+ ports:
+ # the port that this service should serve on
+ - port: 6379
+ targetPort: 6379
+ selector:
+ name: redis-master
diff --git a/cluster/juju/layers/kubernetes/files/guestbook-example/redis-slave-controller.yaml b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-slave-controller.yaml
new file mode 100644
index 00000000000..6e5dde18aa7
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-slave-controller.yaml
@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: redis-slave
+ labels:
+ name: redis-slave
+spec:
+ replicas: 2
+ selector:
+ name: redis-slave
+ template:
+ metadata:
+ labels:
+ name: redis-slave
+ spec:
+ containers:
+ - name: worker
+ image: gcr.io/google_samples/gb-redisslave:v1
+ env:
+ - name: GET_HOSTS_FROM
+ value: dns
+ # If your cluster config does not include a dns service, then to
+ # instead access an environment variable to find the master
+ # service's host, comment out the 'value: dns' line above, and
+ # uncomment the line below.
+ # value: env
+ ports:
+ - containerPort: 6379
diff --git a/cluster/juju/layers/kubernetes/files/guestbook-example/redis-slave-service.yaml b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-slave-service.yaml
new file mode 100644
index 00000000000..1acc9def301
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/files/guestbook-example/redis-slave-service.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: redis-slave
+ labels:
+ name: redis-slave
+spec:
+ ports:
+ # the port that this service should serve on
+ - port: 6379
+ selector:
+ name: redis-slave
diff --git a/cluster/juju/layers/kubernetes/icon.svg b/cluster/juju/layers/kubernetes/icon.svg
new file mode 100644
index 00000000000..55098d1079f
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/icon.svg
@@ -0,0 +1,270 @@
+
+
+
+
diff --git a/cluster/juju/layers/kubernetes/layer.yaml b/cluster/juju/layers/kubernetes/layer.yaml
new file mode 100644
index 00000000000..dfdeebdcec9
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/layer.yaml
@@ -0,0 +1 @@
+includes: ['layer:docker', 'layer:flannel', 'layer:tls', 'interface:etcd']
diff --git a/cluster/juju/layers/kubernetes/metadata.yaml b/cluster/juju/layers/kubernetes/metadata.yaml
new file mode 100644
index 00000000000..cd83a08c08c
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/metadata.yaml
@@ -0,0 +1,17 @@
+name: kubernetes
+summary: Kubernetes is an application container orchestration platform.
+maintainers:
+ - Matthew Bruzek
+ - Charles Butler
+description: |
+ Kubernetes is an open-source platform for deplying, scaling, and operations
+ of appliation containers across a cluster of hosts. Kubernetes is portable
+ in that it works with public, private, and hybrid clouds. Extensible through
+ a pluggable infrastructure. Self healing in that it will automatically
+ restart and place containers on healthy nodes if a node ever goes away.
+tags:
+ - infrastructure
+subordinate: false
+requires:
+ etcd:
+ interface: etcd
diff --git a/cluster/juju/layers/kubernetes/reactive/k8s.py b/cluster/juju/layers/kubernetes/reactive/k8s.py
new file mode 100644
index 00000000000..17c94e6c63b
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/reactive/k8s.py
@@ -0,0 +1,321 @@
+import os
+
+from shlex import split
+from shutil import copy2
+from subprocess import check_call
+
+from charms.docker.compose import Compose
+from charms.reactive import hook
+from charms.reactive import remove_state
+from charms.reactive import set_state
+from charms.reactive import when
+from charms.reactive import when_not
+
+from charmhelpers.core import hookenv
+from charmhelpers.core.hookenv import is_leader
+from charmhelpers.core.hookenv import status_set
+from charmhelpers.core.templating import render
+from charmhelpers.core import unitdata
+from charmhelpers.core.host import chdir
+from contextlib import contextmanager
+
+
+@hook('config-changed')
+def config_changed():
+ '''If the configuration values change, remove the available states.'''
+ config = hookenv.config()
+ if any(config.changed(key) for key in config.keys()):
+ hookenv.log('Configuration options have changed.')
+ # Use the Compose class that encapsulates the docker-compose commands.
+ compose = Compose('files/kubernetes')
+ hookenv.log('Removing kubelet container and kubelet.available state.')
+ # Stop and remove the Kubernetes kubelet container..
+ compose.kill('kubelet')
+ compose.rm('kubelet')
+ # Remove the state so the code can react to restarting kubelet.
+ remove_state('kubelet.available')
+ hookenv.log('Removing proxy container and proxy.available state.')
+ # Stop and remove the Kubernetes proxy container.
+ compose.kill('proxy')
+ compose.rm('proxy')
+ # Remove the state so the code can react to restarting proxy.
+ remove_state('proxy.available')
+
+ if config.changed('version'):
+ hookenv.log('Removing kubectl.downloaded state so the new version'
+ ' of kubectl will be downloaded.')
+ remove_state('kubectl.downloaded')
+
+
+@when('tls.server.certificate available')
+@when_not('k8s.server.certificate available')
+def server_cert():
+ '''When the server certificate is available, get the server certificate from
+ the charm unit data and write it to the proper directory. '''
+ destination_directory = '/srv/kubernetes'
+ # Save the server certificate from unitdata to /srv/kubernetes/server.crt
+ save_certificate(destination_directory, 'server')
+ # Copy the unitname.key to /srv/kubernetes/server.key
+ copy_key(destination_directory, 'server')
+ set_state('k8s.server.certificate available')
+
+
+@when('tls.client.certificate available')
+@when_not('k8s.client.certficate available')
+def client_cert():
+ '''When the client certificate is available, get the client certificate
+ from the charm unitdata and write it to the proper directory. '''
+ destination_directory = '/srv/kubernetes'
+ if not os.path.isdir(destination_directory):
+ os.makedirs(destination_directory)
+ os.chmod(destination_directory, 0o770)
+ # The client certificate is also available on charm unitdata.
+ client_cert_path = 'easy-rsa/easyrsa3/pki/issued/client.crt'
+ kube_cert_path = os.path.join(destination_directory, 'client.crt')
+ if os.path.isfile(client_cert_path):
+ # Copy the client.crt to /srv/kubernetes/client.crt
+ copy2(client_cert_path, kube_cert_path)
+ # The client key is only available on the leader.
+ client_key_path = 'easy-rsa/easyrsa3/pki/private/client.key'
+ kube_key_path = os.path.join(destination_directory, 'client.key')
+ if os.path.isfile(client_key_path):
+ # Copy the client.key to /srv/kubernetes/client.key
+ copy2(client_key_path, kube_key_path)
+
+
+@when('tls.certificate.authority available')
+@when_not('k8s.certificate.authority available')
+def ca():
+ '''When the Certificate Authority is available, copy the CA from the
+ /usr/local/share/ca-certificates/k8s.crt to the proper directory. '''
+ # Ensure the /srv/kubernetes directory exists.
+ directory = '/srv/kubernetes'
+ if not os.path.isdir(directory):
+ os.makedirs(directory)
+ os.chmod(directory, 0o770)
+ # Normally the CA is just on the leader, but the tls layer installs the
+ # CA on all systems in the /usr/local/share/ca-certificates directory.
+ ca_path = '/usr/local/share/ca-certificates/{0}.crt'.format(
+ hookenv.service_name())
+ # The CA should be copied to the destination directory and named 'ca.crt'.
+ destination_ca_path = os.path.join(directory, 'ca.crt')
+ if os.path.isfile(ca_path):
+ copy2(ca_path, destination_ca_path)
+ set_state('k8s.certificate.authority available')
+
+
+@when('kubelet.available', 'proxy.available', 'cadvisor.available')
+def final_messaging():
+ '''Lower layers emit messages, and if we do not clear the status messaging
+ queue, we are left with whatever the last method call sets status to. '''
+ # It's good UX to have consistent messaging that the cluster is online
+ if is_leader():
+ status_set('active', 'Kubernetes leader running')
+ else:
+ status_set('active', 'Kubernetes follower running')
+
+
+@when('kubelet.available', 'proxy.available', 'cadvisor.available')
+@when_not('skydns.available')
+def launch_skydns():
+ '''Create a kubernetes service and resource controller for the skydns
+ service. '''
+ # Only launch and track this state on the leader.
+ # Launching duplicate SkyDNS rc will raise an error
+ if not is_leader():
+ return
+ cmd = "kubectl create -f files/manifests/skydns-rc.yml"
+ check_call(split(cmd))
+ cmd = "kubectl create -f files/manifests/skydns-svc.yml"
+ check_call(split(cmd))
+ set_state('skydns.available')
+
+
+@when('docker.available')
+@when_not('etcd.available')
+def relation_message():
+ '''Take over messaging to let the user know they are pending a relationship
+ to the ETCD cluster before going any further. '''
+ status_set('waiting', 'Waiting for relation to ETCD')
+
+
+@when('etcd.available', 'tls.server.certificate available')
+@when_not('kubelet.available', 'proxy.available')
+def master(etcd):
+ '''Install and run the hyperkube container that starts kubernetes-master.
+ This actually runs the kubelet, which in turn runs a pod that contains the
+ other master components. '''
+ render_files(etcd)
+ # Use the Compose class that encapsulates the docker-compose commands.
+ compose = Compose('files/kubernetes')
+ status_set('maintenance', 'Starting the Kubernetes kubelet container.')
+ # Start the Kubernetes kubelet container using docker-compose.
+ compose.up('kubelet')
+ set_state('kubelet.available')
+ # Open the secure port for api-server.
+ hookenv.open_port(6443)
+ status_set('maintenance', 'Starting the Kubernetes proxy container')
+ # Start the Kubernetes proxy container using docker-compose.
+ compose.up('proxy')
+ set_state('proxy.available')
+ status_set('active', 'Kubernetes started')
+
+
+@when('proxy.available')
+@when_not('kubectl.downloaded')
+def download_kubectl():
+ '''Download the kubectl binary to test and interact with the cluster.'''
+ status_set('maintenance', 'Downloading the kubectl binary')
+ version = hookenv.config()['version']
+ cmd = 'wget -nv -O /usr/local/bin/kubectl https://storage.googleapis.com/' \
+ 'kubernetes-release/release/{0}/bin/linux/amd64/kubectl'
+ cmd = cmd.format(version)
+ hookenv.log('Downloading kubelet: {0}'.format(cmd))
+ check_call(split(cmd))
+ cmd = 'chmod +x /usr/local/bin/kubectl'
+ check_call(split(cmd))
+ set_state('kubectl.downloaded')
+ status_set('active', 'Kubernetes installed')
+
+
+@when('kubectl.downloaded')
+@when_not('kubectl.package.created')
+def package_kubectl():
+ '''Package the kubectl binary and configuration to a tar file for users
+ to consume and interact directly with Kubernetes.'''
+ if not is_leader():
+ return
+ context = 'default-context'
+ cluster_name = 'kubernetes'
+ public_address = hookenv.unit_public_ip()
+ directory = '/srv/kubernetes'
+ key = 'client.key'
+ ca = 'ca.crt'
+ cert = 'client.crt'
+ user = 'ubuntu'
+ port = '6443'
+ with chdir(directory):
+ # Create the config file with the external address for this server.
+ cmd = 'kubectl config set-cluster --kubeconfig={0}/config {1} ' \
+ '--server=https://{2}:{3} --certificate-authority={4}'
+ check_call(split(cmd.format(directory, cluster_name, public_address,
+ port, ca)))
+ # Create the credentials.
+ cmd = 'kubectl config set-credentials --kubeconfig={0}/config {1} ' \
+ '--client-key={2} --client-certificate={3}'
+ check_call(split(cmd.format(directory, user, key, cert)))
+ # Create a default context with the cluster.
+ cmd = 'kubectl config set-context --kubeconfig={0}/config {1}' \
+ ' --cluster={2} --user={3}'
+ check_call(split(cmd.format(directory, context, cluster_name, user)))
+ # Now make the config use this new context.
+ cmd = 'kubectl config use-context --kubeconfig={0}/config {1}'
+ check_call(split(cmd.format(directory, context)))
+ # Copy the kubectl binary to this directory
+ cmd = 'cp -v /usr/local/bin/kubectl {0}'.format(directory)
+ check_call(split(cmd))
+
+ # Create an archive with all the necessary files.
+ cmd = 'tar -cvzf /home/ubuntu/kubectl_package.tar.gz ca.crt client.crt client.key config kubectl' # noqa
+ check_call(split(cmd))
+ set_state('kubectl.package.created')
+
+
+@when('proxy.available')
+@when_not('cadvisor.available')
+def start_cadvisor():
+ '''Start the cAdvisor container that gives metrics about the other
+ application containers on this system. '''
+ compose = Compose('files/kubernetes')
+ compose.up('cadvisor')
+ set_state('cadvisor.available')
+ status_set('active', 'cadvisor running on port 8088')
+ hookenv.open_port(8088)
+
+
+@when('sdn.available')
+def gather_sdn_data():
+ '''Get the Software Defined Network (SDN) information and return it as a
+ dictionary.'''
+ # SDN Providers pass data via the unitdata.kv module
+ db = unitdata.kv()
+ # Generate an IP address for the DNS provider
+ subnet = db.get('sdn_subnet')
+ if subnet:
+ ip = subnet.split('/')[0]
+ dns_server = '.'.join(ip.split('.')[0:-1]) + '.10'
+ addedcontext = {}
+ addedcontext['dns_server'] = dns_server
+ return addedcontext
+ return {}
+
+
+def copy_key(directory, prefix):
+ '''Copy the key from the easy-rsa/easyrsa3/pki/private directory to the
+ specified directory. '''
+ if not os.path.isdir(directory):
+ os.makedirs(directory)
+ os.chmod(directory, 0o770)
+ # Must remove the path characters from the local unit name.
+ path_name = hookenv.local_unit().replace('/', '_')
+ # The key is not in unitdata it is in the local easy-rsa directory.
+ local_key_path = 'easy-rsa/easyrsa3/pki/private/{0}.key'.format(path_name)
+ key_name = '{0}.key'.format(prefix)
+ # The key should be copied to this directory.
+ destination_key_path = os.path.join(directory, key_name)
+ # Copy the key file from the local directory to the destination.
+ copy2(local_key_path, destination_key_path)
+
+
+def render_files(reldata=None):
+ '''Use jinja templating to render the docker-compose.yml and master.json
+ file to contain the dynamic data for the configuration files.'''
+ context = {}
+ # Load the context manager with sdn and config data.
+ context.update(gather_sdn_data())
+ context.update(hookenv.config())
+ if reldata:
+ context.update({'connection_string': reldata.connection_string()})
+ charm_dir = hookenv.charm_dir()
+ rendered_kube_dir = os.path.join(charm_dir, 'files/kubernetes')
+ if not os.path.exists(rendered_kube_dir):
+ os.makedirs(rendered_kube_dir)
+ rendered_manifest_dir = os.path.join(charm_dir, 'files/manifests')
+ if not os.path.exists(rendered_manifest_dir):
+ os.makedirs(rendered_manifest_dir)
+ # Add the manifest directory so the docker-compose file can have.
+ context.update({'manifest_directory': rendered_manifest_dir,
+ 'private_address': hookenv.unit_get('private-address')})
+
+ # Render the files/kubernetes/docker-compose.yml file that contains the
+ # definition for kubelet and proxy.
+ target = os.path.join(rendered_kube_dir, 'docker-compose.yml')
+ render('docker-compose.yml', target, context)
+ # Render the files/manifests/master.json that contains parameters for the
+ # apiserver, controller, and controller-manager
+ target = os.path.join(rendered_manifest_dir, 'master.json')
+ render('master.json', target, context)
+ # Render files/kubernetes/skydns-svc.yaml for SkyDNS service
+ target = os.path.join(rendered_manifest_dir, 'skydns-svc.yml')
+ render('skydns-svc.yml', target, context)
+ # Render files/kubernetes/skydns-rc.yaml for SkyDNS pods
+ target = os.path.join(rendered_manifest_dir, 'skydns-rc.yml')
+ render('skydns-rc.yml', target, context)
+
+
+def save_certificate(directory, prefix):
+ '''Get the certificate from the charm unitdata, and write it to the proper
+ directory. The parameters are: destination directory, and prefix to use
+ for the key and certificate name.'''
+ if not os.path.isdir(directory):
+ os.makedirs(directory)
+ os.chmod(directory, 0o770)
+ # Grab the unitdata key value store.
+ store = unitdata.kv()
+ certificate_data = store.get('tls.{0}.certificate'.format(prefix))
+ certificate_name = '{0}.crt'.format(prefix)
+ # The certificate should be saved to this directory.
+ certificate_path = os.path.join(directory, certificate_name)
+ # write the server certificate out to the correct location
+ with open(certificate_path, 'w') as fp:
+ fp.write(certificate_data)
diff --git a/cluster/juju/layers/kubernetes/templates/docker-compose.yml b/cluster/juju/layers/kubernetes/templates/docker-compose.yml
new file mode 100644
index 00000000000..1beb7c4f6a7
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/templates/docker-compose.yml
@@ -0,0 +1,78 @@
+# https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/docker.md
+
+# docker run \
+# --volume=/:/rootfs:ro \
+# --volume=/sys:/sys:ro \
+# --volume=/dev:/dev \
+# --volume=/var/lib/docker/:/var/lib/docker:rw \
+# --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \
+# --volume=/var/run:/var/run:rw \
+# --volume=/var/lib/juju/agents/unit-k8s-0/charm/files/manifests:/etc/kubernetes/manifests:rw \
+# --volume=/srv/kubernetes:/srv/kubernetes \
+# --net=host \
+# --pid=host \
+# --privileged=true \
+# -ti \
+# gcr.io/google_containers/hyperkube:v1.0.6 \
+# /hyperkube kubelet --containerized --hostname-override="127.0.0.1" \
+# --address="0.0.0.0" --api-servers=http://localhost:8080 \
+# --config=/etc/kubernetes/manifests
+
+kubelet:
+ image: gcr.io/google_containers/hyperkube:{{version}}
+ net: host
+ pid: host
+ privileged: true
+ restart: always
+ volumes:
+ - /:/rootfs:ro
+ - /sys:/sys:ro
+ - /dev:/dev
+ - /var/lib/docker/:/var/lib/docker:rw
+ - /var/lib/kubelet/:/var/lib/kubelet:rw
+ - /var/run:/var/run:rw
+ - {{manifest_directory}}:/etc/kubernetes/manifests:rw
+ - /srv/kubernetes:/srv/kubernetes
+ command: |
+ /hyperkube kubelet --containerized --hostname-override="{{private_address}}"
+ --address="0.0.0.0" --api-servers=http://localhost:8080
+ --config=/etc/kubernetes/manifests {% if dns_server %}
+ --cluster-dns={{dns_server}} --cluster-domain=cluster.local {% endif %}
+ --tls-cert-file="/srv/kubernetes/server.crt"
+ --tls-private-key-file="/srv/kubernetes/server.key"
+
+# docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 \
+# /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001 \
+# --data-dir=/var/etcd/data
+etcd:
+ net: host
+ image: gcr.io/google_containers/etcd:2.0.12
+ command: |
+ /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001
+ --data-dir=/var/etcd/data
+
+# docker run \
+# -d \
+# --net=host \
+# --privileged \
+# gcr.io/google_containers/hyperkube:v${K8S_VERSION} \
+# /hyperkube proxy --master=http://127.0.0.1:8080 --v=2
+proxy:
+ net: host
+ privileged: true
+ restart: always
+ image: gcr.io/google_containers/hyperkube:{{version}}
+ command: /hyperkube proxy --master=http://127.0.0.1:8080 --v=2
+
+# cAdvisor (Container Advisor) provides container users an understanding of
+# the resource usage and performance characteristics of their running containers.
+cadvisor:
+ image: google/cadvisor:latest
+ volumes:
+ - /:/rootfs:ro
+ - /var/run:/var/run:rw
+ - /sys:/sys:ro
+ - /var/lib/docker:/var/lib/docker:ro
+ ports:
+ - 8088:8080
+ restart: always
diff --git a/cluster/juju/layers/kubernetes/templates/master.json b/cluster/juju/layers/kubernetes/templates/master.json
new file mode 100644
index 00000000000..52f010911e4
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/templates/master.json
@@ -0,0 +1,61 @@
+{
+"apiVersion": "v1",
+"kind": "Pod",
+"metadata": {"name":"k8s-master"},
+"spec":{
+ "hostNetwork": true,
+ "containers":[
+ {
+ "name": "controller-manager",
+ "image": "gcr.io/google_containers/hyperkube:{{version}}",
+ "command": [
+ "/hyperkube",
+ "controller-manager",
+ "--master=127.0.0.1:8080",
+ "--v=2"
+ ]
+ },
+ {
+ "name": "apiserver",
+ "image": "gcr.io/google_containers/hyperkube:{{version}}",
+ "command": [
+ "/hyperkube",
+ "apiserver",
+ "--address=0.0.0.0",
+ "--client_ca_file=/srv/kubernetes/ca.crt",
+ "--cluster-name=kubernetes",
+ "--etcd-servers={{connection_string}}",
+ "--service-cluster-ip-range={{cidr}}",
+ "--tls-cert-file=/srv/kubernetes/server.crt",
+ "--tls-private-key-file=/srv/kubernetes/server.key",
+ "--v=2"
+ ],
+ "volumeMounts": [
+ {
+ "mountPath": "/srv/kubernetes",
+ "name": "certs-kubernetes",
+ "readOnly": true
+ }
+ ]
+ },
+ {
+ "name": "scheduler",
+ "image": "gcr.io/google_containers/hyperkube:{{version}}",
+ "command": [
+ "/hyperkube",
+ "scheduler",
+ "--master=127.0.0.1:8080",
+ "--v=2"
+ ]
+ }
+ ],
+ "volumes": [
+ {
+ "hostPath": {
+ "path": "/srv/kubernetes"
+ },
+ "name": "certs-kubernetes"
+ }
+ ]
+ }
+}
diff --git a/cluster/juju/layers/kubernetes/templates/skydns-rc.yml b/cluster/juju/layers/kubernetes/templates/skydns-rc.yml
new file mode 100644
index 00000000000..d151c20948a
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/templates/skydns-rc.yml
@@ -0,0 +1,92 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: kube-dns-v8
+ namespace: kube-system
+ labels:
+ k8s-app: kube-dns
+ version: v8
+ kubernetes.io/cluster-service: "true"
+spec:
+ {% if dns_replicas -%} replicas: {{ dns_replicas }} {% else %} replicas: 1 {% endif %}
+ selector:
+ k8s-app: kube-dns
+ version: v8
+ template:
+ metadata:
+ labels:
+ k8s-app: kube-dns
+ version: v8
+ kubernetes.io/cluster-service: "true"
+ spec:
+ containers:
+ - name: etcd
+ image: gcr.io/google_containers/etcd:2.0.9
+ resources:
+ limits:
+ cpu: 100m
+ memory: 50Mi
+ command:
+ - /usr/local/bin/etcd
+ - -data-dir
+ - /var/etcd/data
+ - -listen-client-urls
+ - http://127.0.0.1:2379,http://127.0.0.1:4001
+ - -advertise-client-urls
+ - http://127.0.0.1:2379,http://127.0.0.1:4001
+ - -initial-cluster-token
+ - skydns-etcd
+ volumeMounts:
+ - name: etcd-storage
+ mountPath: /var/etcd/data
+ - name: kube2sky
+ image: gcr.io/google_containers/kube2sky:1.11
+ resources:
+ limits:
+ cpu: 100m
+ memory: 50Mi
+ args:
+ # command = "/kube2sky"
+ {% if dns_domain -%}- -domain={{ dns_domain }} {% else %} - -domain=cluster.local {% endif %}
+ - -kube_master_url=http://{{private_address}}:8080
+ - name: skydns
+ image: gcr.io/google_containers/skydns:2015-03-11-001
+ resources:
+ limits:
+ cpu: 100m
+ memory: 50Mi
+ args:
+ # command = "/skydns"
+ - -machines=http://localhost:4001
+ - -addr=0.0.0.0:53
+ {% if dns_domain -%}- -domain={{ dns_domain }}. {% else %} - -domain=cluster.local. {% endif %}
+ ports:
+ - containerPort: 53
+ name: dns
+ protocol: UDP
+ - containerPort: 53
+ name: dns-tcp
+ protocol: TCP
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: 8080
+ scheme: HTTP
+ initialDelaySeconds: 30
+ timeoutSeconds: 5
+ - name: healthz
+ image: gcr.io/google_containers/exechealthz:1.0
+ resources:
+ limits:
+ cpu: 10m
+ memory: 20Mi
+ args:
+ {% if dns_domain -%}- -cmd=nslookup kubernetes.default.svc.{{ pillar['dns_domain'] }} localhost >/dev/null {% else %} - -cmd=nslookup kubernetes.default.svc.kubernetes.local localhost >/dev/null {% endif %}
+ - -port=8080
+ ports:
+ - containerPort: 8080
+ protocol: TCP
+ volumes:
+ - name: etcd-storage
+ emptyDir: {}
+ dnsPolicy: Default # Don't use cluster DNS.
diff --git a/cluster/juju/layers/kubernetes/templates/skydns-svc.yml b/cluster/juju/layers/kubernetes/templates/skydns-svc.yml
new file mode 100644
index 00000000000..ec0689ada18
--- /dev/null
+++ b/cluster/juju/layers/kubernetes/templates/skydns-svc.yml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: kube-dns
+ namespace: kube-system
+ labels:
+ k8s-app: kube-dns
+ kubernetes.io/cluster-service: "true"
+ kubernetes.io/name: "KubeDNS"
+spec:
+ selector:
+ k8s-app: kube-dns
+ clusterIP: {{ dns_server }}
+ ports:
+ - name: dns
+ port: 53
+ protocol: UDP
+ - name: dns-tcp
+ port: 53
+ protocol: TCP
diff --git a/cluster/juju/prereqs/ubuntu-juju.sh b/cluster/juju/prereqs/ubuntu-juju.sh
index c85245942b4..a0921eb3917 100644
--- a/cluster/juju/prereqs/ubuntu-juju.sh
+++ b/cluster/juju/prereqs/ubuntu-juju.sh
@@ -43,6 +43,6 @@ function gather_installation_reqs() {
sudo apt-get update
fi
- package_status 'juju-quickstart'
- package_status 'juju-deployer'
+ package_status 'juju'
+ package_status 'charm-tools'
}
diff --git a/cluster/juju/util.sh b/cluster/juju/util.sh
index f8113437dff..fed49f0d062 100755
--- a/cluster/juju/util.sh
+++ b/cluster/juju/util.sh
@@ -25,38 +25,28 @@ JUJU_PATH=$(dirname ${UTIL_SCRIPT})
KUBE_ROOT=$(readlink -m ${JUJU_PATH}/../../)
# Use the config file specified in $KUBE_CONFIG_FILE, or config-default.sh.
source "${JUJU_PATH}/${KUBE_CONFIG_FILE-config-default.sh}"
+# This attempts installation of Juju - This really needs to support multiple
+# providers/distros - but I'm super familiar with ubuntu so assume that for now.
source ${JUJU_PATH}/prereqs/ubuntu-juju.sh
export JUJU_REPOSITORY=${JUJU_PATH}/charms
-#KUBE_BUNDLE_URL='https://raw.githubusercontent.com/whitmo/bundle-kubernetes/master/bundles.yaml'
KUBE_BUNDLE_PATH=${JUJU_PATH}/bundles/local.yaml
-# Build the binaries on the local system and copy the binaries to the Juju charm.
+
function build-local() {
- local targets=(
- cmd/kube-proxy \
- cmd/kube-apiserver \
- cmd/kube-controller-manager \
- cmd/kubelet \
- plugin/cmd/kube-scheduler \
- cmd/kubectl \
- test/e2e/e2e.test \
- )
- # Make a clean environment to avoid compiler errors.
- make clean
- # Build the binaries locally that are used in the charms.
- make all WHAT="${targets[*]}"
- local OUTPUT_DIR=_output/local/bin/linux/amd64
- mkdir -p cluster/juju/charms/trusty/kubernetes-master/files/output
- # Copy the binaries from the output directory to the charm directory.
- cp -v $OUTPUT_DIR/* cluster/juju/charms/trusty/kubernetes-master/files/output
+ # This used to build the kubernetes project. Now it rebuilds the charm(s)
+ # living in `cluster/juju/layers`
+
+ charm build -o $JUJU_REPOSITORY -s trusty ${JUJU_PATH}/layers/kubernetes
}
function detect-master() {
local kubestatus
+
# Capturing a newline, and my awk-fu was weak - pipe through tr -d
- kubestatus=$(juju status --format=oneline kubernetes-master | grep kubernetes-master/0 | awk '{print $3}' | tr -d "\n")
+ kubestatus=$(juju status --format=oneline kubernetes | grep ${KUBE_MASTER_NAME} | awk '{print $3}' | tr -d "\n")
export KUBE_MASTER_IP=${kubestatus}
- export KUBE_SERVER=http://${KUBE_MASTER_IP}:8080
+ export KUBE_SERVER=https://${KUBE_MASTER_IP}:6433
+
}
function detect-nodes() {
@@ -74,25 +64,14 @@ function detect-nodes() {
export NUM_NODES=${#KUBE_NODE_IP_ADDRESSES[@]}
}
-function get-password() {
- export KUBE_USER=admin
- # Get the password from the basic-auth.csv file on kubernetes-master.
- export KUBE_PASSWORD=$(juju run --unit kubernetes-master/0 "cat /srv/kubernetes/basic-auth.csv" | grep ${KUBE_USER} | cut -d, -f1)
-}
-
function kube-up() {
build-local
- if [[ -d "~/.juju/current-env" ]]; then
- juju quickstart -i --no-browser
- else
- juju quickstart --no-browser
- fi
+
# The juju-deployer command will deploy the bundle and can be run
# multiple times to continue deploying the parts that fail.
- juju deployer -c ${KUBE_BUNDLE_PATH}
+ juju deploy ${KUBE_BUNDLE_PATH}
source "${KUBE_ROOT}/cluster/common.sh"
- get-password
# Sleep due to juju bug http://pad.lv/1432759
sleep-status
@@ -100,31 +79,22 @@ function kube-up() {
detect-nodes
local prefix=$RANDOM
- export KUBE_CERT="/tmp/${prefix}-kubecfg.crt"
- export KUBE_KEY="/tmp/${prefix}-kubecfg.key"
- export CA_CERT="/tmp/${prefix}-kubecfg.ca"
- export CONTEXT="juju"
-
+ export KUBECONFIG=/tmp/${prefix}/config
# Copy the cert and key to this machine.
(
umask 077
- juju scp kubernetes-master/0:/srv/kubernetes/apiserver.crt ${KUBE_CERT}
- juju run --unit kubernetes-master/0 'chmod 644 /srv/kubernetes/apiserver.key'
- juju scp kubernetes-master/0:/srv/kubernetes/apiserver.key ${KUBE_KEY}
- juju run --unit kubernetes-master/0 'chmod 600 /srv/kubernetes/apiserver.key'
- cp ${KUBE_CERT} ${CA_CERT}
-
- create-kubeconfig
+ mkdir -p /tmp/${prefix}
+ juju scp ${KUBE_MASTER_NAME}:kubectl_package.tar.gz /tmp/${prefix}/
+ ls -al /tmp/${prefix}/
+ tar xfz /tmp/${prefix}/kubectl_package.tar.gz -C /tmp/${prefix}
)
}
function kube-down() {
local force="${1-}"
- # Remove the binary files from the charm directory.
- rm -rf cluster/juju/charms/trusty/kubernetes-master/files/output/
local jujuenv
- jujuenv=$(cat ~/.juju/current-environment)
- juju destroy-environment ${jujuenv} ${force} || true
+ jujuenv=$(juju switch)
+ juju destroy-model ${jujuenv} ${force} || true
}
function prepare-e2e() {
@@ -140,23 +110,13 @@ function sleep-status() {
jujustatus=''
echo "Waiting up to 15 minutes to allow the cluster to come online... wait for it..." 1>&2
- jujustatus=$(juju status kubernetes-master --format=oneline)
- if [[ $jujustatus == *"started"* ]];
- then
- return
- fi
-
- while [[ $i < $maxtime && $jujustatus != *"started"* ]]; do
- sleep 15
- i+=15
- jujustatus=$(juju status kubernetes-master --format=oneline)
+ while [[ $i < $maxtime && -z $jujustatus ]]; do
+ sleep 15
+ i+=15
+ jujustatus=$(${JUJU_PATH}/identify-leaders.py)
+ export KUBE_MASTER_NAME=${jujustatus}
done
- # sleep because we cannot get the status back of where the minions are in the deploy phase
- # thanks to a generic "started" state and our service not actually coming online until the
- # minions have received the binary from the master distribution hub during relations
- echo "Sleeping an additional minute to allow the cluster to settle" 1>&2
- sleep 60
}
# Execute prior to running tests to build a release if required for environment.
diff --git a/cluster/kubectl.sh b/cluster/kubectl.sh
index 77ad5a13515..30a57760570 100755
--- a/cluster/kubectl.sh
+++ b/cluster/kubectl.sh
@@ -116,7 +116,7 @@ kubectl="${KUBECTL_PATH:-${kubectl}}"
if [[ "$KUBERNETES_PROVIDER" == "gke" ]]; then
detect-project &> /dev/null
-elif [[ "$KUBERNETES_PROVIDER" == "ubuntu" || "$KUBERNETES_PROVIDER" == "juju" ]]; then
+elif [[ "$KUBERNETES_PROVIDER" == "ubuntu" ]]; then
detect-master > /dev/null
config=(
"--server=http://${KUBE_MASTER_IP}:8080"