Launch a cluster-local registry.

This registry can be accessed through proxies that run on each node
listening on port 5000. We send the proxy images to the nodes directly
to avoid requests that hit the network during cluster launch. For now,
we continue to pull the registry itself over the network, especially
given its large size (we should be able to dramatically shrink the
image). On GCE we create a PD and use that for storage, otherwise we
use an emptyDir. The registry is not enabled outside of GCE. All
communication is currently plain HTTP. In order to use SSL, we will
need to be able to request a certificate/key from the apiserver signed
by the apiserver's CA cert.

cluster/saltbase/install.sh

@@ -64,7 +64,9 @@ done
 echo "+++ Install binaries from tar: $1"
 tar -xz -C "${KUBE_TEMP}" -f "$1"
 mkdir -p /srv/salt-new/salt/kube-bins
+mkdir -p /srv/salt-new/salt/kube-addons-images
 cp -v "${KUBE_TEMP}/kubernetes/server/bin/"* /srv/salt-new/salt/kube-bins/
+cp -v "${KUBE_TEMP}/kubernetes/addons/"* /srv/salt-new/salt/kube-addons-images/
 
 kube_bin_dir="/srv/salt-new/salt/kube-bins";
 docker_images_sls_file="/srv/salt-new/pillar/docker-images.sls";

cluster/saltbase/salt/kube-addons/init.sls

@@ -81,6 +81,25 @@ addon-dir-create:
     - makedirs: True
 {% endif %}
 
+{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %}
+/etc/kubernetes/addons/registry/registry-svc.yaml:
+  file.managed:
+    - source: salt://kube-addons/registry/registry-svc.yaml
+    - user: root
+    - group: root
+    - file_mode: 644
+    - makedirs: True
+
+/etc/kubernetes/addons/registry/registry-rc.yaml:
+  file.managed:
+    - source: salt://kube-addons/registry/registry-rc.yaml.in
+    - template: jinja
+    - user: root
+    - group: root
+    - file_mode: 644
+    - makedirs: True
+{% endif %}
+
 {% if pillar.get('enable_node_logging', '').lower() == 'true'
    and pillar.get('logging_destination').lower() == 'elasticsearch'
    and pillar.get('enable_cluster_logging', '').lower() == 'true' %}

cluster/saltbase/salt/kube-addons/kube-addons.sh

@@ -125,6 +125,28 @@ function create-resource-from-string() {
   return 1;
 }
 
+# $1 is the directory containing all of the docker images
+function load-docker-images() {
+  local success
+  local restart_docker
+  while true; do
+    success=true
+    restart_docker=false
+    for image in "$1/"*; do
+      timeout 30 docker load -i "${image}" &>/dev/null
+      rc=$?
+      if [[ $rc == 124 ]]; then
+        restart_docker=true
+      elif [[ $rc != 0 ]]; then
+        success=false
+      fi
+    done
+    if [[ $success == true ]]; then break; fi
+    if [[ $restart_docker == true ]]; then service docker restart; fi
+    sleep 15
+  done
+}
+
 # The business logic for whether a given object should be created
 # was already enforced by salt, and /etc/kubernetes/addons is the
 # managed result is of that. Start everything below that directory.
@@ -142,6 +164,9 @@ for k,v in yaml.load(sys.stdin).iteritems():
 ''' < "${kube_env_yaml}")
 fi
 
+# Load any images that we may need
+load-docker-images /srv/salt/kube-addons-images
+
 # Create the namespace that will be used to host the cluster-level add-ons.
 start_addon /etc/kubernetes/addons/namespace.yaml 100 10 "" &
 

cluster/saltbase/salt/kube-registry-proxy/init.sls

@@ -0,0 +1,8 @@
+/etc/kubernetes/manifests/kube-registry-proxy.yaml:
+  file.managed:
+    - source: salt://kube-registry-proxy/kube-registry-proxy.yaml
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: True
+    - dir_mode: 755

cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-registry-proxy
+  namespace: kube-system
+spec:
+  containers:
+  - name: kube-registry-proxy
+    image: uluyol/kube-registry-proxy:0.2.3
+    resources:
+      limits:
+        cpu: 100m
+        memory: 50Mi
+    env:
+    - name: REGISTRY_HOST
+      value: kube-registry.kube-system.svc.cluster.local
+    - name: REGISTRY_PORT
+      value: "5000"
+    - name: FORWARD_PORT
+      value: "5000"
+    ports:
+    - name: registry
+      containerPort: 5000
+      hostPort: 5000

cluster/saltbase/salt/top.sls

@@ -24,6 +24,9 @@ base:
   {% elif pillar['logging_destination'] == 'gcp' %}
     - fluentd-gcp
   {% endif %}
+{% endif %}
+{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %}
+    - kube-registry-proxy
 {% endif %}
     - logrotate
 {% if grains['cloud'] is defined and grains.cloud == 'gce' %}