26c431affa
Automatic merge from submit-queue (batch tested with PRs 45268, 47573, 47632, 47818)
NODE_TAINTS in gce startup scripts
Currently there is now way to pass a list of taints that should be added on node registration (at least not in gce or other saltbased deployment). This PR adds necessary plumbing to pass the taints from user or instance group template to kubelet startup flags.
```release-note
Taints support in gce/salt startup scripts.
```
The PR was manually tested.
```
NODE_TAINTS: 'dedicated=ml:NoSchedule'
```
in kube-env results in
```
spec:
[...]
taints:
- effect: NoSchedule
key: dedicated
timeAdded: null
value: ml
```
cc: @davidopp @gmarek @dchen1107 @MaciekPytel
193 lines
8.4 KiB
Plaintext
193 lines
8.4 KiB
Plaintext
{% set daemon_args = "$DAEMON_ARGS" -%}
|
|
{% if grains['os_family'] == 'RedHat' -%}
|
|
{% set daemon_args = "" -%}
|
|
{% endif -%}
|
|
|
|
{% if grains.api_servers is defined -%}
|
|
{% set api_servers = "--api-servers=https://" + grains.api_servers -%}
|
|
{% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form
|
|
{% set api_servers = "--api-servers=https://" + grains.apiservers -%}
|
|
{% elif grains['roles'][0] == 'kubernetes-master' -%}
|
|
{% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%}
|
|
{% set api_servers = "--api-servers=https://" + master_ipv4 -%}
|
|
{% else -%}
|
|
{% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
|
|
{% set api_servers = "--api-servers=https://" + ips[0][0] -%}
|
|
{% endif -%}
|
|
|
|
{% set master_kubelet_args = "" %}
|
|
|
|
{% set debugging_handlers = "--enable-debugging-handlers=true" -%}
|
|
|
|
{% if grains['roles'][0] == 'kubernetes-master' -%}
|
|
{% if grains.cloud in ['aws', 'gce', 'vagrant', 'photon-controller', 'openstack', 'azure-legacy'] -%}
|
|
|
|
# Unless given a specific directive, disable registration for the kubelet
|
|
# running on the master.
|
|
{% if grains.kubelet_api_servers is defined -%}
|
|
{% set api_servers = "--api-servers=https://" + grains.kubelet_api_servers -%}
|
|
{% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%}
|
|
{% else -%}
|
|
{% set api_servers = "" -%}
|
|
{% endif -%}
|
|
|
|
# Disable the debugging handlers (/run and /exec) to prevent arbitrary
|
|
# code execution on the master.
|
|
# TODO(roberthbailey): Relax this constraint once the master is self-hosted.
|
|
{% set debugging_handlers = "--enable-debugging-handlers=false" -%}
|
|
{% endif -%}
|
|
{% endif -%}
|
|
|
|
{% if grains.cloud == 'gce' -%}
|
|
{% set api_servers = "--bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig --require-kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig" -%}
|
|
{% endif -%}
|
|
|
|
{% set cloud_provider = "" -%}
|
|
{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%}
|
|
{% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
|
|
{% endif -%}
|
|
|
|
{% set cloud_config = "" -%}
|
|
{% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%}
|
|
{% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
|
|
{% endif -%}
|
|
|
|
{% set config = "--pod-manifest-path=/etc/kubernetes/manifests" -%}
|
|
|
|
{% set manifest_url = "" -%}
|
|
{% set manifest_url_header = "" -%}
|
|
{% if pillar.get('enable_manifest_url', '').lower() == 'true' %}
|
|
{% set manifest_url = "--manifest-url=" + pillar['manifest_url'] + " --manifest-url-header=" + pillar['manifest_url_header'] -%}
|
|
{% endif -%}
|
|
|
|
{% set hostname_override = "" -%}
|
|
{% if grains.hostname_override is defined -%}
|
|
{% set hostname_override = " --hostname-override=" + grains.hostname_override -%}
|
|
{% endif -%}
|
|
|
|
{% set cluster_dns = "" %}
|
|
{% set cluster_domain = "" %}
|
|
{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %}
|
|
{% set cluster_dns = "--cluster-dns=" + pillar['dns_server'] %}
|
|
{% set cluster_domain = "--cluster-domain=" + pillar['dns_domain'] %}
|
|
{% endif %}
|
|
|
|
{% set docker_root = "" -%}
|
|
{% if grains.docker_root is defined -%}
|
|
{% set docker_root = " --docker-root=" + grains.docker_root -%}
|
|
{% endif -%}
|
|
|
|
{% set kubelet_root = "" -%}
|
|
{% if grains.kubelet_root is defined -%}
|
|
{% set kubelet_root = " --root-dir=" + grains.kubelet_root -%}
|
|
{% endif -%}
|
|
|
|
{% set non_masquerade_cidr = "" -%}
|
|
{% if pillar.get('non_masquerade_cidr','') -%}
|
|
{% set non_masquerade_cidr = "--non-masquerade-cidr=" + pillar.non_masquerade_cidr -%}
|
|
{% endif -%}
|
|
|
|
# Setup cgroups hierarchies.
|
|
{% set cgroup_root = "" -%}
|
|
{% set system_container = "" -%}
|
|
{% set kubelet_container = "" -%}
|
|
{% set runtime_container = "" -%}
|
|
{% if grains['os_family'] == 'Debian' -%}
|
|
{% if pillar.get('is_systemd') %}
|
|
{% set cgroup_root = "--cgroup-root=docker" -%}
|
|
{% else %}
|
|
{% set cgroup_root = "--cgroup-root=/" -%}
|
|
{% set system_container = "--system-cgroups=/system" -%}
|
|
{% set runtime_container = "--runtime-cgroups=/docker-daemon" -%}
|
|
{% set kubelet_container= "--kubelet-cgroups=/kubelet" -%}
|
|
{% endif %}
|
|
{% endif -%}
|
|
{% if grains['oscodename'] in ['vivid','wily'] -%}
|
|
{% set cgroup_root = "--cgroup-root=docker" -%}
|
|
{% endif -%}
|
|
|
|
{% set pod_cidr = "" %}
|
|
{% if grains['roles'][0] == 'kubernetes-master' %}
|
|
{% if grains.get('cbr-cidr') %}
|
|
{% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %}
|
|
{% elif api_servers == '' and pillar.get('network_provider', '').lower() == 'kubenet' %}
|
|
# Kubelet standalone mode needs a PodCIDR since there is no controller-manager
|
|
{% set pod_cidr = "--pod-cidr=10.76.0.0/16" %}
|
|
{% endif -%}
|
|
{% endif %}
|
|
|
|
{% set cpu_cfs_quota = "" %}
|
|
{% if pillar['enable_cpu_cfs_quota'] is defined -%}
|
|
{% set cpu_cfs_quota = "--cpu-cfs-quota=" + pillar['enable_cpu_cfs_quota'] -%}
|
|
{% endif -%}
|
|
|
|
{% set feature_gates = "" -%}
|
|
{% if grains['feature_gates'] is defined -%}
|
|
{% set feature_gates = "--feature-gates=" + grains['feature_gates'] -%}
|
|
{% endif %}
|
|
|
|
{% set test_args = "" -%}
|
|
{% if pillar['kubelet_test_args'] is defined -%}
|
|
{% set test_args=pillar['kubelet_test_args'] %}
|
|
{% endif -%}
|
|
|
|
{% set network_plugin = "" -%}
|
|
{% if pillar.get('network_provider', '').lower() == 'opencontrail' %}
|
|
{% set network_plugin = "--network-plugin=opencontrail" %}
|
|
{% elif pillar.get('network_provider', '').lower() == 'cni' %}
|
|
{% set network_plugin = "--network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %}
|
|
{% elif pillar.get('network_policy_provider', '').lower() == 'calico' and grains['roles'][0] != 'kubernetes-master' %}
|
|
{% set network_plugin = "--network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %}
|
|
{% elif pillar.get('network_provider', '').lower() == 'kubenet' %}
|
|
{% set network_plugin = "--network-plugin=kubenet --network-plugin-dir=/home/kubernetes/bin/" -%}
|
|
{% endif -%}
|
|
|
|
# Don't pipe the --hairpin-mode flag by default. This allows the kubelet to pick
|
|
# an appropriate value.
|
|
{% set hairpin_mode = "" -%}
|
|
# The master cannot see Services because it doesn't run kube-proxy, so we don't
|
|
# need to make its container bridge promiscuous. We also don't want to set
|
|
# the hairpin-veth flag on the master because it increases the chances of
|
|
# running into the kernel bug described in #20096.
|
|
{% if grains['roles'][0] == 'kubernetes-master' -%}
|
|
{% set hairpin_mode = "--hairpin-mode=none" -%}
|
|
{% elif pillar['hairpin_mode'] is defined and pillar['hairpin_mode'] in ['promiscuous-bridge', 'hairpin-veth', 'none'] -%}
|
|
{% set hairpin_mode = "--hairpin-mode=" + pillar['hairpin_mode'] -%}
|
|
{% endif -%}
|
|
|
|
{% set kubelet_port = "" -%}
|
|
{% if pillar['kubelet_port'] is defined -%}
|
|
{% set kubelet_port="--port=" + pillar['kubelet_port'] %}
|
|
{% endif -%}
|
|
|
|
{% set log_level = pillar['log_level'] -%}
|
|
{% if pillar['kubelet_test_log_level'] is defined -%}
|
|
{% set log_level = pillar['kubelet_test_log_level'] -%}
|
|
{% endif -%}
|
|
|
|
{% set enable_custom_metrics = "" -%}
|
|
{% if pillar['enable_custom_metrics'] is defined -%}
|
|
{% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %}
|
|
{% endif -%}
|
|
|
|
{% set node_labels = "" %}
|
|
{% if pillar['node_labels'] is defined -%}
|
|
{% set node_labels="--node-labels=" + pillar['node_labels'] %}
|
|
{% endif -%}
|
|
|
|
{% set node_taints = "" %}
|
|
{% if pillar['node_taints'] is defined -%}
|
|
{% set node_taints="--register-with-taints=" + pillar['node_taints'] %}
|
|
{% endif -%}
|
|
|
|
{% set eviction_hard = "" %}
|
|
{% if pillar['eviction_hard'] is defined -%}
|
|
{% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %}
|
|
{% endif -%}
|
|
|
|
{% set kubelet_auth = "--anonymous-auth=false --authorization-mode=Webhook --client-ca-file=" + pillar.get('ca_cert_bundle_path', '/var/lib/kubelet/ca.crt') %}
|
|
{% set pki=" --cert-dir=/var/lib/kubelet/pki" -%}
|
|
|
|
# test_args has to be kept at the end, so they'll overwrite any prior configuration
|
|
DAEMON_ARGS="{{daemon_args}} {{api_servers}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{node_taints}} {{eviction_hard}} {{kubelet_auth}} {{pki}} {{feature_gates}} {{test_args}}"
|