Give the API server access to TLS certs.

Moved the cert generation to a separate salt state and put it in a more appropriate sharable location (`/srv/kubernetes/`).

cluster/aws/util.sh

@@ -227,7 +227,7 @@ function kube-up {
   if [ ! -f $AWS_SSH_KEY ]; then
     ssh-keygen -f $AWS_SSH_KEY -N ''
   fi
-    
+
   $AWS_CMD import-key-pair --key-name kubernetes --public-key-material file://$AWS_SSH_KEY.pub > /dev/null 2>&1 || true
   VPC_ID=$($AWS_CMD create-vpc --cidr-block 172.20.0.0/16 | json_val '["Vpc"]["VpcId"]')
   $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-support '{"Value": true}' > /dev/null
@@ -294,14 +294,14 @@ function kube-up {
       --security-group-ids $SEC_GROUP_ID \
       --associate-public-ip-address \
       --user-data file://${KUBE_TEMP}/minion-start-${i}.sh | json_val '["Instances"][0]["InstanceId"]')
-    sleep 3        
+    sleep 3
     n=0
     until [ $n -ge 5 ]; do
       $AWS_CMD create-tags --resources $minion_id --tags Key=Name,Value=${MINION_NAMES[$i]} > /dev/null && break
       n=$[$n+1]
       sleep 15
     done
-        
+
     sleep 3
     n=0
     until [ $n -ge 5 ]; do
@@ -309,7 +309,7 @@ function kube-up {
       n=$[$n+1]
       sleep 15
     done
-    
+
     sleep 3
     $AWS_CMD modify-instance-attribute --instance-id $minion_id --source-dest-check '{"Value": false}' > /dev/null
 
@@ -343,7 +343,7 @@ function kube-up {
   detect-master > /dev/null
   detect-minions > /dev/null
 
-  # Wait 3 minutes for cluster to come up.  We hit it with a "highstate" after that to 
+  # Wait 3 minutes for cluster to come up.  We hit it with a "highstate" after that to
   # make sure that everything is well configured.
   echo "Waiting for cluster to settle"
   local i
@@ -353,7 +353,7 @@ function kube-up {
   done
   echo "Re-running salt highstate"
   ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo salt '*' state.highstate > /dev/null
-    
+
   echo "Waiting for cluster initialization."
   echo
   echo "  This will continually check to see if the API for kubernetes is reachable."
@@ -400,9 +400,9 @@ function kube-up {
   # config file.  Distribute the same way the htpasswd is done.
   (
     umask 077
-    ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
+    ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
-    ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
+    ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
-    ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
+    ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
 
     cat << EOF > ~/.kubernetes_auth
 {

cluster/gce/util.sh

@@ -422,9 +422,9 @@ function kube-up {
   # TODO: generate ADMIN (and KUBELET) tokens and put those in the master's
   # config file.  Distribute the same way the htpasswd is done.
   (umask 077
-   gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
+   gcutil ssh "${MASTER_NAME}" sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
-   gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
+   gcutil ssh "${MASTER_NAME}" sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
-   gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
+   gcutil ssh "${MASTER_NAME}" sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
 
    cat << EOF > ~/.kubernetes_auth
 {

cluster/saltbase/salt/apiserver/default

@@ -27,4 +27,7 @@
   {% set portal_net = "-portal_net=" + pillar['portal_net'] %}
 {% endif %}
 
-DAEMON_ARGS="{{daemon_args}} {{address}} {{etcd_servers}} {{ cloud_provider }} --allow_privileged={{pillar['allow_privileged']}} {{portal_net}}"
+{% set cert_file = "-tls_cert_file=/srv/kubernetes/server.cert" %}
+{% set key_file = "-tls_private_key_file=/srv/kubernetes/server.key" %}
+
+DAEMON_ARGS="{{daemon_args}} {{address}} {{etcd_servers}} {{ cloud_provider }} --allow_privileged={{pillar['allow_privileged']}} {{portal_net}} {{cert_file}} {{key_file}}"

cluster/saltbase/salt/generate-cert/init.sls

@@ -0,0 +1,38 @@
+{% if grains.cloud is defined %}
+  {% if grains.cloud == 'gce' %}
+    {% set cert_ip='_use_gce_external_ip_' %}
+  {% endif %}
+  {% if grains.cloud == 'aws' %}
+    {% set cert_ip='_use_aws_external_ip_' %}
+  {% endif %}
+  {% if grains.cloud == 'vagrant' %}
+    {% set cert_ip=grains.fqdn_ip4 %}
+  {% endif %}
+  {% if grains.cloud == 'vsphere' %}
+    {% set cert_ip=grains.ip_interfaces.eth0[0] %}
+  {% endif %}
+{% endif %}
+
+# If there is a pillar defined, override any defaults.
+{% if pillar['cert_ip'] is defined %}
+  {% set cert_ip=pillar['cert_ip'] %}
+{% endif %}
+
+{% set certgen="make-cert.sh" %}
+{% if cert_ip is defined %}
+  {% set certgen="make-ca-cert.sh" %}
+{% endif %}
+
+kubernetes-cert:
+  cmd.script:
+    - unless: test -f /srv/kubernetes/server.cert
+    - source: salt://generate-cert/{{certgen}}
+{% if cert_ip is defined %}
+    - args: {{cert_ip}}
+    - require:
+      - pkg: curl
+{% endif %}
+    - cwd: /
+    - user: root
+    - group: root
+    - shell: /bin/bash

cluster/saltbase/salt/generate-cert/make-ca-cert.sh

@@ -19,6 +19,9 @@ set -o nounset
 set -o pipefail
 
 cert_ip=$1
+cert_dir=/srv/kubernetes
+
+mkdir -p "$cert_dir"
 
 # TODO: Add support for discovery on other providers?
 if [ "$cert_ip" == "_use_gce_external_ip_" ]; then
@@ -33,19 +36,28 @@ tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
 trap 'rm -rf "${tmpdir}"' EXIT
 cd "${tmpdir}"
 
-# TODO: For now, this is a patched repo that makes subject-alt-name work, when the fix is upstream
+# TODO: For now, this is a patched tool that makes subject-alt-name work, when
-#  move back to the upstream easyrsa
+# the fix is upstream  move back to the upstream easyrsa.  This is cached in GCS
-curl -L -J -O https://github.com/brendandburns/easy-rsa/archive/master.tar.gz > /dev/null 2>&1
+# but is originally taken from:
-tar xzf easy-rsa-master.tar.gz > /dev/null 2>&1
+#   https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
+#
+# To update, do the following:
+# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
+# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
+# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
+#
+# Due to GCS caching of public objects, it may take time for this to be widely
+# distributed.
+curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
+tar xzf easy-rsa.tar.gz > /dev/null 2>&1
 
 cd easy-rsa-master/easyrsa3
 ./easyrsa init-pki > /dev/null 2>&1
 ./easyrsa --batch build-ca nopass > /dev/null 2>&1
 ./easyrsa --subject-alt-name=IP:$cert_ip build-server-full kubernetes-master nopass > /dev/null 2>&1
 ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1
-cp -p pki/issued/kubernetes-master.crt /usr/share/nginx/server.cert > /dev/null 2>&1
+cp -p pki/issued/kubernetes-master.crt "${cert_dir}/server.cert" > /dev/null 2>&1
-cp -p pki/private/kubernetes-master.key /usr/share/nginx/server.key > /dev/null 2>&1
+cp -p pki/private/kubernetes-master.key "${cert_dir}/server.key" > /dev/null 2>&1
-cp -p pki/ca.crt /usr/share/nginx/ca.crt
+cp -p pki/ca.crt "${cert_dir}/ca.crt"
-cp -p pki/issued/kubecfg.crt /usr/share/nginx/kubecfg.crt
+cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
-cp -p pki/private/kubecfg.key /usr/share/nginx/kubecfg.key
+cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
-

cluster/saltbase/salt/generate-cert/make-cert.sh

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+cert_dir=/srv/kubernetes
+
 openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
   -subj "/CN=kubernetes.invalid/O=Kubernetes" \
-  -keyout /usr/share/nginx/server.key  -out /usr/share/nginx/server.cert
+  -keyout "${cert_dir}/server.key" -out "${cert_dir}/server.cert"

cluster/saltbase/salt/nginx/init.sls

@@ -8,45 +8,7 @@ nginx:
       - file: /etc/nginx/nginx.conf
       - file: /etc/nginx/sites-enabled/default
       - file: /usr/share/nginx/htpasswd
-      - cmd: /usr/share/nginx/server.cert
+      - cmd: kubernetes-cert
-
-{% if grains.cloud is defined %}
-  {% if grains.cloud == 'gce' %}
-    {% set cert_ip='_use_gce_external_ip_' %}
-  {% endif %}
-  {% if grains.cloud == 'aws' %}
-    {% set cert_ip='_use_aws_external_ip_' %}
-  {% endif %}
-  {% if grains.cloud == 'vagrant' %}
-    {% set cert_ip=grains.fqdn_ip4 %}
-  {% endif %}
-  {% if grains.cloud == 'vsphere' %}
-    {% set cert_ip=grains.ip_interfaces.eth0[0] %}
-  {% endif %}
-{% endif %}
-# If there is a pillar defined, override any defaults.
-{% if pillar['cert_ip'] is defined %}
-  {% set cert_ip=pillar['cert_ip'] %}
-{% endif %}
-
-{% set certgen="make-cert.sh" %}
-{% if cert_ip is defined %}
-  {% set certgen="make-ca-cert.sh" %}
-{% endif %}
-
-/usr/share/nginx/server.cert:
-  cmd.script:
-    - unless: test -f /usr/share/nginx/server.cert
-    - source: salt://nginx/{{certgen}} 
-{% if cert_ip is defined %}
-    - args: {{cert_ip}}
-    - require:
-      - pkg: curl
-{% endif %}
-    - cwd: /
-    - user: root
-    - group: root
-    - shell: /bin/bash
 
 /etc/nginx/nginx.conf:
   file:

cluster/saltbase/salt/nginx/kubernetes-site

@@ -33,8 +33,8 @@ server {
         index index.html index.htm;
 
         ssl on;
-        ssl_certificate /usr/share/nginx/server.cert;
+        ssl_certificate /srv/kubernetes/server.cert;
-        ssl_certificate_key /usr/share/nginx/server.key;
+        ssl_certificate_key /srv/kubernetes/server.key;
 
         ssl_session_timeout 5m;
 
@@ -53,7 +53,7 @@ server {
           proxy_connect_timeout 159s;
           proxy_send_timeout   600s;
           proxy_read_timeout   600s;
-          
+
           # Disable retry
           proxy_next_upstream off;
 

cluster/saltbase/salt/top.sls

@@ -26,6 +26,7 @@ base:
 
   'roles:kubernetes-master':
     - match: grain
+    - generate-cert
     - etcd
     - apiserver
     - controller-manager

cluster/vsphere/util.sh

@@ -397,9 +397,9 @@ function kube-up {
   (
     umask 077
 
-    kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
+    kube-ssh "${KUBE_MASTER_IP}" sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
-    kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
+    kube-ssh "${KUBE_MASTER_IP}" sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
-    kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
+    kube-ssh "${KUBE_MASTER_IP}" sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
 
     cat << EOF > ~/.kubernetes_auth
     {