Merge pull request #6618 from roberthbailey/no-nginx

Salt reconfiguration to get rid of nginx on GCE

cluster/common.sh

@@ -28,6 +28,7 @@ DEFAULT_KUBECONFIG="${HOME}/.kube/config"
 # Assumed vars:
 #   KUBE_USER
 #   KUBE_PASSWORD
+#   KUBE_BEARER_TOKEN
 #   KUBE_MASTER_IP
 #   KUBECONFIG
 #   CONTEXT
@@ -56,10 +57,17 @@ function create-kubeconfig() {
       "--embed-certs=true"
     )
   fi
-  local user_args=(
+  local user_args=()
+  if [[ -z "${KUBE_USER:-}" || -z "${KUBE_PASSWORD:-}" ]]; then
+    user_args+=(
+     "--token=${KUBE_BEARER_TOKEN}"
+    )
+  else
+    user_args+=(
      "--username=${KUBE_USER}"
      "--password=${KUBE_PASSWORD}"
-  )
+    )
+  fi
   if [[ ! -z "${KUBE_CERT:-}" && ! -z "${KUBE_KEY:-}" ]]; then
     user_args+=(
      "--client-certificate=${KUBE_CERT}"
@@ -124,3 +132,28 @@ function get-kubeconfig-basicauth() {
     KUBE_PASSWORD=''
   fi
 }
+
+# Get the bearer token for the current-context in kubeconfig if one exists.
+# Assumed vars:
+#   KUBECONFIG  # if unset, defaults to global
+#
+# Vars set:
+#   KUBE_BEARER_TOKEN
+#
+# KUBE_BEARER_TOKEN will be empty if no current-context is set, or the
+# current-context user does not exist or contain a bearer token entry.
+function get-kubeconfig-bearertoken() {
+  export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
+  # Template to safely extract the token for the current-context user.
+  # The long chain of 'with' commands avoids indexing nil if any of the
+  # entries ("current-context", "contexts"."current-context", "users", etc)
+  # is missing.
+  # Note: we save dot ('.') to $root because the 'with' action overrides it.
+  # See http://golang.org/pkg/text/template/.
+  local token='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{range $element := (index $dot "contexts")}}{{ if eq .name $ctx }}{{ with $user := .context.user }}{{range $element := (index $dot "users")}}{{ if eq .name $user }}{{ index . "user" "token" }}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}'
+  KUBE_BEARER_TOKEN=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${token}")
+  # Handle empty/missing token
+  if [[ "${KUBE_BEARER_TOKEN}" == '<no value>' ]]; then
+    KUBE_BEARER_TOKEN=''
+  fi
+}

cluster/gce/configure-vm.sh

@@ -251,23 +251,17 @@ EOF
 }
 
 # This should only happen on cluster initialization. Uses
-# MASTER_HTPASSWORD to generate the nginx/htpasswd file, and the
-# KUBELET_TOKEN, plus /dev/urandom, to generate known_tokens.csv
-# (KNOWN_TOKENS_FILE). After the first boot and on upgrade, these
-# files exist on the master-pd and should never be touched again
-# (except perhaps an additional service account, see NB below.)
+# KUBE_BEARER_TOKEN, KUBELET_TOKEN, and /dev/urandom to generate
+# known_tokens.csv (KNOWN_TOKENS_FILE). After the first boot and
+# on upgrade, this file exists on the master-pd and should never
+# be touched again (except perhaps an additional service account,
+# see NB below.)
 function create-salt-auth() {
-  local -r htpasswd_file="/srv/salt-overlay/salt/nginx/htpasswd"
-
-  if [ ! -e "${htpasswd_file}" ]; then
-    mkdir -p /srv/salt-overlay/salt/nginx
-    echo "${MASTER_HTPASSWD}" > "${htpasswd_file}"
-  fi
-
   if [ ! -e "${KNOWN_TOKENS_FILE}" ]; then
     mkdir -p /srv/salt-overlay/salt/kube-apiserver
     (umask 077;
-      echo "${KUBELET_TOKEN},kubelet,kubelet" > "${KNOWN_TOKENS_FILE}")
+      echo "${KUBE_BEARER_TOKEN},admin,admin" > "${KNOWN_TOKENS_FILE}";
+      echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${KNOWN_TOKENS_FILE}")
 
     mkdir -p /srv/salt-overlay/salt/kubelet
     kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"

cluster/gce/util.sh

@@ -272,30 +272,21 @@ function get-password {
   fi
 }
 
-# Set MASTER_HTPASSWD
-function set-master-htpasswd {
-  python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
-    -b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
-  local htpasswd
-  MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
-}
-
-# Generate authentication token for admin user. Will
-# read from $HOME/.kubernetes_auth if available.
+# Ensure that we have a bearer token created for validating to the master.
+# Will read from kubeconfig for the current context if available.
+#
+# Assumed vars
+#   KUBE_ROOT
 #
 # Vars set:
-#   KUBE_ADMIN_TOKEN
-function get-admin-token {
-  local file="$HOME/.kubernetes_auth"
-  if [[ -r "$file" ]]; then
-    KUBE_ADMIN_TOKEN=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["BearerToken"]')
-    return
+#   KUBE_BEARER_TOKEN
+function get-bearer-token() {
+  get-kubeconfig-bearertoken
+  if [[ -z "${KUBE_BEARER_TOKEN:-}" ]]; then
+    KUBE_BEARER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
   fi
-  KUBE_ADMIN_TOKEN=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))')
 }
 
-
-
 # Wait for background jobs to finish. Exit with
 # an error status if any of the jobs failed.
 function wait-for-jobs {
@@ -482,7 +473,7 @@ ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
 DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
 DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
 DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
-MASTER_HTPASSWD: $(yaml-quote ${MASTER_HTPASSWD})
+KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN})
 ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
 MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
 EOF
@@ -516,8 +507,7 @@ function write-node-env {
 # variables are set:
 #   ensure-temp-dir
 #   detect-project
-#   get-password
-#   set-master-htpasswd
+#   get-bearer-token
 #
 function create-master-instance {
   local address_opt=""
@@ -550,8 +540,7 @@ function kube-up {
   ensure-temp-dir
   detect-project
 
-  get-password
-  set-master-htpasswd
+  get-bearer-token
 
   # Make sure we have the tar files staged on Google Storage
   find-release-tars
@@ -679,8 +668,9 @@ function kube-up {
   echo "  up."
   echo
 
-  until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
-          --fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
+  until curl --insecure -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
+          --max-time 5 --fail --output /dev/null --silent \
+          "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do
       printf "."
       sleep 2
   done
@@ -859,8 +849,7 @@ function kube-push {
   detect-project
   detect-master
   detect-minion-names
-  get-password
-  set-master-htpasswd
+  get-bearer-token
 
   # Make sure we have the tar files staged on Google Storage
   find-release-tars

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -44,7 +44,11 @@
 {% set cert_file = "--tls_cert_file=/srv/kubernetes/server.cert" -%}
 {% set key_file = "--tls_private_key_file=/srv/kubernetes/server.key" -%}
 
-{% set secure_port = "--secure_port=6443" -%}
+{% set secure_port = "6443" -%}
+{% if grains['cloud'] is defined and grains['cloud'] == 'gce' %}
+  {% set secure_port = "443" -%}
+{% endif -%}
+
 {% set token_auth_file = "--token_auth_file=/dev/null" -%}
 
 {% if grains.cloud is defined -%}
@@ -77,24 +81,24 @@
                  "/kube-apiserver",
                  "{{address}}",
                  "{{etcd_servers}}",
-                 "{{ cloud_provider }}",
-                 "{{ cloud_config }}",
-                 "{{ runtime_config }}",
+                 "{{cloud_provider}}",
+                 "{{cloud_config}}",
+                 "{{runtime_config}}",
                  "{{admission_control}}",
                  "--allow_privileged={{pillar['allow_privileged']}}",
                  "{{portal_net}}",
                  "{{cluster_name}}",
                  "{{cert_file}}",
                  "{{key_file}}",
-                 "{{secure_port}}",
+                 "--secure_port={{secure_port}}",
                  "{{token_auth_file}}",
                  "{{publicAddressOverride}}",
                  "{{pillar['log_level']}}"
                ],
     "ports":[
       { "name": "https",
-        "containerPort": 6443,
-        "hostPort": 6443},{
+        "containerPort": {{secure_port}},
+        "hostPort": {{secure_port}}},{
        "name": "http",
         "containerPort": 7080,
         "hostPort": 7080},{

cluster/saltbase/salt/kubelet/default

@@ -4,15 +4,22 @@
 {% endif -%}
 
 {% if grains.api_servers is defined -%}
-  {% set api_servers = "--api_servers=https://" + grains.api_servers + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + grains.api_servers -%}
 {% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form
-  {% set api_servers = "--api_servers=https://" + grains.apiservers + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + grains.apiservers -%}
 {% elif grains['roles'][0] == 'kubernetes-master' -%}
   {% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%}
-  {% set api_servers = "--api_servers=https://" + master_ipv4 + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + master_ipv4 -%}
 {% else -%}
   {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
-  {% set api_servers = "--api_servers=https://" + ips[0][0] + ":6443" -%}
+  {% set api_servers = "--api_servers=https://" + ips[0][0] -%}
+{% endif -%}
+
+# TODO: remove nginx for other cloud providers.
+{% if grains['cloud'] is defined and grains['cloud'] == 'gce' -%}
+  {% set api_servers_with_port = api_servers -%}
+{% else -%}
+  {% set api_servers_with_port = api_servers + ":6443" -%}
 {% endif -%}
 
 {% set config = "--config=/etc/kubernetes/manifests" -%}
@@ -33,4 +40,4 @@
   {% set docker_root = " --docker_root=" + grains.docker_root -%}
 {% endif -%}
 
-DAEMON_ARGS="{{daemon_args}} {{api_servers}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"
+DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}}"

cluster/saltbase/salt/top.sls

@@ -33,7 +33,9 @@ base:
     - kube-controller-manager
     - kube-scheduler
     - monit
+{% if grains['cloud'] is defined and grains['cloud'] != 'gce' %}
     - nginx
+{% endif %}
     - cadvisor
     - kube-client-tools
     - kube-master-addons