From 5a0159ea00e082bc85bbec18d1ab7ae78d90fa4f Mon Sep 17 00:00:00 2001
From: Joe Beda <joe.github@bedafamily.com>
Date: Thu, 13 Nov 2014 22:14:56 -0800
Subject: [PATCH] Get Vagrant to start using TLS client certs.

Also fix up cert generation. It was failing during the first salt highstate when trying to chown the certs as the apiserver user didn't exist yet.  Fix this by creating a 'kube-cert' group and chgrping the files to that.  Then make the apiserver a member of that group.

Fixes #2365
Fixes #2368
---
 cluster/kubecfg.sh                            |  9 +------
 cluster/kubectl.sh                            |  7 ------
 cluster/saltbase/salt/apiserver/init.sls      |  2 ++
 cluster/saltbase/salt/generate-cert/init.sls  |  6 ++++-
 .../salt/generate-cert/make-ca-cert.sh        |  5 ++--
 .../saltbase/salt/generate-cert/make-cert.sh  |  5 ++--
 cluster/saltbase/salt/nginx/init.sls          | 18 +++++++------
 cluster/vagrant/provision-master.sh           |  2 ++
 cluster/vagrant/util.sh                       | 25 ++++++++++++++++++-
 docs/salt.md                                  |  2 +-
 hack/e2e-suite/goe2e.sh                       | 11 +++++++-
 11 files changed, 61 insertions(+), 31 deletions(-)

diff --git a/cluster/kubecfg.sh b/cluster/kubecfg.sh
index 228d71a494e..72cc2ce75f5 100755
--- a/cluster/kubecfg.sh
+++ b/cluster/kubecfg.sh
@@ -83,16 +83,9 @@ fi
 
 # When we are using vagrant it has hard coded auth.  We repeat that here so that
 # we don't clobber auth that might be used for a publicly facing cluster.
-if [ "$KUBERNETES_PROVIDER" == "vagrant" ]; then
-  cat >~/.kubernetes_vagrant_auth <<EOF
-{
-  "User": "vagrant",
-  "Password": "vagrant"
-}
-EOF
+if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]]; then
   auth_config=(
     "-auth" "$HOME/.kubernetes_vagrant_auth"
-    "-insecure_skip_tls_verify"
   )
 else
   auth_config=()
diff --git a/cluster/kubectl.sh b/cluster/kubectl.sh
index 11c73891bdc..a460b32af25 100755
--- a/cluster/kubectl.sh
+++ b/cluster/kubectl.sh
@@ -84,15 +84,8 @@ fi
 # When we are using vagrant it has hard coded auth.  We repeat that here so that
 # we don't clobber auth that might be used for a publicly facing cluster.
 if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]]; then
-  cat >~/.kubernetes_vagrant_auth <<EOF
-{
-  "User": "vagrant",
-  "Password": "vagrant"
-}
-EOF
   auth_config=(
     "--auth-path=$HOME/.kubernetes_vagrant_auth"
-    "--insecure-skip-tls-verify=true"
   )
 else
   auth_config=()
diff --git a/cluster/saltbase/salt/apiserver/init.sls b/cluster/saltbase/salt/apiserver/init.sls
index 2d0909caedb..e0deee5dcc5 100644
--- a/cluster/saltbase/salt/apiserver/init.sls
+++ b/cluster/saltbase/salt/apiserver/init.sls
@@ -44,6 +44,8 @@ apiserver:
   user.present:
     - system: True
     - gid_from_name: True
+    - groups:
+      - kube-cert
     - shell: /sbin/nologin
     - home: /var/apiserver
     - require:
diff --git a/cluster/saltbase/salt/generate-cert/init.sls b/cluster/saltbase/salt/generate-cert/init.sls
index 250d7e6763b..604685a5269 100644
--- a/cluster/saltbase/salt/generate-cert/init.sls
+++ b/cluster/saltbase/salt/generate-cert/init.sls
@@ -6,7 +6,7 @@
     {% set cert_ip='_use_aws_external_ip_' %}
   {% endif %}
   {% if grains.cloud == 'vagrant' %}
-    {% set cert_ip=grains.fqdn_ip4 %}
+    {% set cert_ip=grains.ip_interfaces.eth1[0] %}
   {% endif %}
   {% if grains.cloud == 'vsphere' %}
     {% set cert_ip=grains.ip_interfaces.eth0[0] %}
@@ -23,6 +23,10 @@
   {% set certgen="make-ca-cert.sh" %}
 {% endif %}
 
+kube-cert:
+  group.present:
+    - system: True
+
 kubernetes-cert:
   cmd.script:
     - unless: test -f /srv/kubernetes/server.cert
diff --git a/cluster/saltbase/salt/generate-cert/make-ca-cert.sh b/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
index f3ab21f37ed..eb6fa3d7161 100755
--- a/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
+++ b/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
@@ -20,7 +20,7 @@ set -o pipefail
 
 cert_ip=$1
 cert_dir=/srv/kubernetes
-cert_file_owner=apiserver.apiserver
+cert_group=kube-cert
 
 mkdir -p "$cert_dir"
 
@@ -63,4 +63,5 @@ cp -p pki/ca.crt "${cert_dir}/ca.crt"
 cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
 cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
 # Make server certs accessible to apiserver.
-chown $cert_file_owner "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.cert"
+chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"
+chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"
diff --git a/cluster/saltbase/salt/generate-cert/make-cert.sh b/cluster/saltbase/salt/generate-cert/make-cert.sh
index 36f7d9d4f01..5a82006590b 100755
--- a/cluster/saltbase/salt/generate-cert/make-cert.sh
+++ b/cluster/saltbase/salt/generate-cert/make-cert.sh
@@ -15,11 +15,12 @@
 # limitations under the License.
 
 cert_dir=/srv/kubernetes
-cert_file_owner=apiserver.apiserver
+cert_group=kube-cert
 
 mkdir -p "$cert_dir"
 
 openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
   -subj "/CN=kubernetes.invalid/O=Kubernetes" \
   -keyout "${cert_dir}/server.key" -out "${cert_dir}/server.cert"
-chown $cert_file_owner "${cert_dir}/server.key" "${cert_dir}/server.cert"
+chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert"
+chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert"
diff --git a/cluster/saltbase/salt/nginx/init.sls b/cluster/saltbase/salt/nginx/init.sls
index bbc4fdeb102..473123c0afc 100644
--- a/cluster/saltbase/salt/nginx/init.sls
+++ b/cluster/saltbase/salt/nginx/init.sls
@@ -1,14 +1,6 @@
 nginx:
   pkg:
     - installed
-  service:
-    - running
-    - watch:
-      - pkg: nginx
-      - file: /etc/nginx/nginx.conf
-      - file: /etc/nginx/sites-enabled/default
-      - file: /usr/share/nginx/htpasswd
-      - cmd: kubernetes-cert
 
 /etc/nginx/nginx.conf:
   file:
@@ -36,3 +28,13 @@ nginx:
     - group: root
     - mode: 644
 
+nginx-service:
+  service:
+    - running
+    - name: nginx
+    - watch:
+      - pkg: nginx
+      - file: /etc/nginx/nginx.conf
+      - file: /etc/nginx/sites-enabled/default
+      - file: /usr/share/nginx/htpasswd
+      - cmd: kubernetes-cert
diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh
index fe75fbc5021..a0e6f95354b 100755
--- a/cluster/vagrant/provision-master.sh
+++ b/cluster/vagrant/provision-master.sh
@@ -70,6 +70,7 @@ grains:
   master_ip: $MASTER_IP
   network_mode: openvswitch
   etcd_servers: $MASTER_IP
+  cloud: vagrant
   cloud_provider: vagrant
   roles:
     - kubernetes-master
@@ -78,6 +79,7 @@ EOF
 mkdir -p /srv/salt-overlay/pillar
 cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
   portal_net: $PORTAL_NET
+  cert_ip: $MASTER_IP
 EOF
 
 # Configure the salt-master
diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh
index a38e40b9368..f7285e11295 100644
--- a/cluster/vagrant/util.sh
+++ b/cluster/vagrant/util.sh
@@ -43,6 +43,29 @@ function kube-up {
   get-password
   vagrant up
 
+  local kube_cert=".kubecfg.vagrant.crt"
+  local kube_key=".kubecfg.vagrant.key"
+  local ca_cert=".kubernetes.vagrant.ca.crt"
+
+  (umask 077
+   vagrant ssh master -- sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null
+   vagrant ssh master -- sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null
+   vagrant ssh master -- sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null
+
+   cat << EOF > ~/.kubernetes_vagrant_auth
+{
+  "User": "$KUBE_USER",
+  "Password": "$KUBE_PASSWORD",
+  "CAFile": "$HOME/$ca_cert",
+  "CertFile": "$HOME/$kube_cert",
+  "KeyFile": "$HOME/$kube_key"
+}
+EOF
+
+   chmod 0600 ~/.kubernetes_auth "${HOME}/${kube_cert}" \
+     "${HOME}/${kube_key}" "${HOME}/${ca_cert}"
+  )
+
   echo "Each machine instance has been created."
   echo "  Now waiting for the Salt provisioning process to complete on each machine."
   echo "  This can take some time based on your network, disk, and cpu speed."
@@ -108,7 +131,7 @@ function kube-up {
   echo
   echo "  https://${KUBE_MASTER_IP}"
   echo
-  echo "The user name and password to use is located in ~/.kubernetes_auth."
+  echo "The user name and password to use is located in ~/.kubernetes_vagrant_auth."
   echo
 }
 
diff --git a/docs/salt.md b/docs/salt.md
index 0e23f6de924..ceb796b3315 100644
--- a/docs/salt.md
+++ b/docs/salt.md
@@ -52,7 +52,7 @@ The following enumerates the set of defined key/value pairs that are supported t
 Key | Value
 ------------- | -------------
 `cbr-cidr` | (Optional) The minion IP address range used for the docker container bridge.
-`cloud` | (Optional) Which IaaS platform is used to host kubernetes, *gce*, *azure*
+`cloud` | (Optional) Which IaaS platform is used to host kubernetes, *gce*, *azure*, *aws*, *vagrant*
 `cloud_provider` | (Optional) The cloud_provider used by apiserver: *gce*, *azure*, *vagrant*
 `etcd_servers` | (Optional) Comma-delimited list of IP addresses the apiserver and kubelet use to reach etcd.  Uses the IP of the first machine in the kubernetes_master role.
 `hostnamef` | (Optional) The full host name of the machine, i.e. hostname -f
diff --git a/hack/e2e-suite/goe2e.sh b/hack/e2e-suite/goe2e.sh
index 4716b60e1ef..26082053ce0 100755
--- a/hack/e2e-suite/goe2e.sh
+++ b/hack/e2e-suite/goe2e.sh
@@ -63,5 +63,14 @@ locations=(
 )
 e2e=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
 
+# When we are using vagrant it has hard coded auth.  We repeat that here so that
+# we don't clobber auth that might be used for a publicly facing cluster.
+if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]]; then
+  auth_config=(
+    "--auth_config=$HOME/.kubernetes_vagrant_auth"
+  )
+else
+  auth_config=()
+fi
 
-"${e2e}" -host="https://${KUBE_MASTER_IP-}"
+"${e2e}" "${auth_config[@]:+${auth_config[@]}}" -host="https://${KUBE_MASTER_IP-}"