Merge pull request #50705 from MrHohn/kube-proxy-ds

Automatic merge from submit-queue (batch tested with PRs 50932, 49610, 51312, 51415, 50705)

Allow running kube-proxy as a DaemonSet when using kube-up.sh on GCE

**What this PR does / why we need it**:
From #23225, this PR adds an option for user to run kube-proxy as a DaemonSet instead of static pods using GCE startup scripts. By default, kube-proxy will run as static pods.

This is the first step for moving kube-proxy into a DaemonSet in GCE, remaining tasks will be tracked on #23225.

**Special notes for your reviewer**:
The last commit are purely for testing out kube-proxy as daemonset via CIs.

cc @kubernetes/sig-network-misc @kubernetes/sig-cluster-lifecycle-misc 

**Release note**:

```release-note
When using kube-up.sh on GCE, user could set env `KUBE_PROXY_DAEMONSET=true` to run kube-proxy as a DaemonSet. kube-proxy is run as static pods by default.
```

cluster/saltbase/salt/kube-proxy/init.sls

@@ -7,6 +7,7 @@
     - makedirs: true
 
 # kube-proxy in a static pod
+{% if pillar.get('kube_proxy_daemonset', '').lower() != 'true' %}
 /etc/kubernetes/manifests/kube-proxy.manifest:
   file.managed:
     - source: salt://kube-proxy/kube-proxy.manifest
@@ -24,6 +25,7 @@
     - require:
       - service: docker
       - service: kubelet
+{% endif %}
 
 /var/log/kube-proxy.log:
   file.managed:

cluster/saltbase/salt/kube-proxy/kube-proxy.manifest

@@ -1,3 +1,6 @@
+# Please keep kube-proxy configuration in-sync with:
+# cluster/addons/kube-proxy/kube-proxy-ds.yaml
+
 {% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
 {% if grains.api_servers is defined -%}
   {% set api_servers = "--master=https://" + grains.api_servers -%}
@@ -35,6 +38,8 @@
 {% set params = log_level + " " + throttles + " " + feature_gates + " " + test_args -%}
 
 {% set container_env = "" -%}
+{% set kube_cache_mutation_detector_env_name = "" -%}
+{% set kube_cache_mutation_detector_env_value = "" -%}
 
 # kube-proxy podspec
 apiVersion: v1
@@ -75,6 +80,8 @@ spec:
     - -c
     - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1
     {{container_env}}
+    {{kube_cache_mutation_detector_env_name}}
+      {{kube_cache_mutation_detector_env_value}}
     securityContext:
       privileged: true
     volumeMounts:

cluster/saltbase/salt/kubelet/default

@@ -164,10 +164,16 @@
   {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %}
 {% endif -%}
 
-{% set node_labels = "" %}
-{% if pillar['node_labels'] is defined -%}
-  {% set node_labels="--node-labels=" + pillar['node_labels'] %}
-{% endif -%}
+{% set kube_proxy_ds_label = "" %}
+{% if grains['roles'][0] != 'kubernetes-master' and pillar.get('kube_proxy_daemonset', '').lower() == 'true' %}
+  # Add kube-proxy daemonset label to node to avoid situation during cluster
+  # upgrade/downgrade when there are two instances of kube-proxy running on a node.
+  {% set kube_proxy_ds_label = "beta.kubernetes.io/kube-proxy-ds-ready=true," %}
+{% endif %}
+{% set node_labels = kube_proxy_ds_label + pillar['node_labels'] %}
+{% if node_labels != "" %}
+  {% set node_labels="--node-labels=" + node_labels %}
+{% endif %}
 
 {% set node_taints = "" %}
 {% if pillar['node_taints'] is defined -%}