{% set daemon_args = "$DAEMON_ARGS" -%}
{% if grains['os_family'] == 'RedHat' -%}
  {% set daemon_args = "" -%}
{% endif -%}


# kubeconfig file
{% set require_kubeconfig = "" %}
{% if grains.kubelet_bootstrap_kubeconfig is defined -%}
  {% set bootstrap_kubeconfig = "--bootstrap-kubeconfig=" + grains.kubelet_bootstrap_kubeconfig -%}
{% else -%}
  {% set bootstrap_kubeconfig = "" -%}
{% endif -%}
{% if grains.kubelet_kubeconfig is defined -%}
  {% set kubeconfig = "--kubeconfig=" + grains.kubelet_kubeconfig -%}
{% else -%}
  {% set kubeconfig = "" -%}
{% endif -%}

{% set master_kubelet_args = "" %}

{% set debugging_handlers = "--enable-debugging-handlers=true" -%}

{% if grains['roles'][0] == 'kubernetes-master' -%}
  {% if grains.cloud in ['aws', 'gce', 'vagrant', 'photon-controller', 'openstack', 'azure-legacy'] -%}
    # Unless given a specific directive, disable registration for the kubelet
    # running on the master.
    {% if kubeconfig != "" -%}
      {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%}
    {% endif -%}

    # Disable the debugging handlers (/run and /exec) to prevent arbitrary
    # code execution on the master.
    # TODO(roberthbailey): Relax this constraint once the master is self-hosted.
    {% set debugging_handlers = "--enable-debugging-handlers=false" -%}
  {% endif -%}
{% endif -%}

{% set cloud_provider = "" -%}
{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%}
  {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
{% endif -%}

{% set cloud_config = "" -%}
{% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%}
  {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
{% endif -%}

{% set config = "--pod-manifest-path=/etc/kubernetes/manifests" -%}

{% set manifest_url = "" -%}
{% set manifest_url_header = "" -%}
{% if pillar.get('enable_manifest_url', '').lower() == 'true' %}
  {% set manifest_url = "--manifest-url=" + pillar['manifest_url'] + " --manifest-url-header=" + pillar['manifest_url_header'] -%}
{% endif -%}

{% set hostname_override = "" -%}
{% if grains.hostname_override is defined -%}
  {% set hostname_override = " --hostname-override=" + grains.hostname_override -%}
{% endif -%}

{% set cluster_dns = "" %}
{% set cluster_domain = "" %}
{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %}
  {% set cluster_dns = "--cluster-dns=" + pillar['dns_server'] %}
  {% set cluster_domain = "--cluster-domain=" + pillar['dns_domain'] %}
{% endif %}

{% set docker_root = "" -%}
{% if grains.docker_root is defined -%}
  {% set docker_root = " --docker-root=" + grains.docker_root -%}
{% endif -%}

{% set kubelet_root = "" -%}
{% if grains.kubelet_root is defined -%}
  {% set kubelet_root = " --root-dir=" + grains.kubelet_root -%}
{% endif -%}

{% set non_masquerade_cidr = "" -%}
{% if pillar.get('non_masquerade_cidr','') -%}
  {% set non_masquerade_cidr = "--non-masquerade-cidr=" + pillar.non_masquerade_cidr -%}
{% endif -%}

# Setup cgroups hierarchies.
{% set cgroup_root = "" -%}
{% set system_container = "" -%}
{% set kubelet_container = "" -%}
{% set runtime_container = "" -%}
{% if grains['os_family'] == 'Debian' -%}
  {% if pillar.get('is_systemd') %}
    {% set cgroup_root = "--cgroup-root=docker" -%}
  {% else %}
    {% set cgroup_root = "--cgroup-root=/" -%}
    {% set system_container = "--system-cgroups=/system" -%}
    {% set runtime_container = "--runtime-cgroups=/docker-daemon" -%}
    {% set kubelet_container= "--kubelet-cgroups=/kubelet" -%}
  {% endif %}
{% endif -%}
{% if grains['oscodename'] in ['vivid','wily'] -%}
  {% set cgroup_root = "--cgroup-root=docker" -%}
{% endif -%}

{% set pod_cidr = "" %}
{% if grains['roles'][0] == 'kubernetes-master' %}
  {% if grains.get('cbr-cidr') %}
    {% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %}
  {% elif kubeconfig == "" and pillar.get('network_provider', '').lower() == 'kubenet' %}
    # Kubelet standalone mode needs a PodCIDR since there is no controller-manager
    {% set pod_cidr = "--pod-cidr=10.76.0.0/16" %}
  {% endif -%}
{% endif %}

{% set cpu_cfs_quota = "" %}
{% if pillar['enable_cpu_cfs_quota'] is defined -%}
 {% set cpu_cfs_quota = "--cpu-cfs-quota=" + pillar['enable_cpu_cfs_quota'] -%}
{% endif -%}

{% set feature_gates = "" -%}
{% if grains['feature_gates'] is defined -%}
  {% set feature_gates = "--feature-gates=" + grains['feature_gates'] -%}
{% endif %}

{% set test_args = "" -%}
{% if pillar['kubelet_test_args'] is defined -%}
  {% set test_args=pillar['kubelet_test_args'] %}
{% endif -%}

{% set network_plugin = "" -%}
{% if pillar.get('network_provider', '').lower() == 'opencontrail' %}
  {% set network_plugin = "--network-plugin=opencontrail" %}
{% elif pillar.get('network_provider', '').lower() == 'cni' %}
  {% set network_plugin = "--network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %}
{% elif pillar.get('network_policy_provider', '').lower() == 'calico' and grains['roles'][0] != 'kubernetes-master' %}
  {% set network_plugin = "--network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %}
{% elif pillar.get('network_provider', '').lower() == 'kubenet' %}
  {% set network_plugin = "--network-plugin=kubenet --network-plugin-dir=/home/kubernetes/bin/" -%}
{% endif -%}

# Don't pipe the --hairpin-mode flag by default. This allows the kubelet to pick
# an appropriate value.
{% set hairpin_mode = "" -%}
# The master cannot see Services because it doesn't run kube-proxy, so we don't
# need to make its container bridge promiscuous. We also don't want to set
# the hairpin-veth flag on the master because it increases the chances of
# running into the kernel bug described in #20096.
{% if grains['roles'][0] == 'kubernetes-master' -%}
  {% set hairpin_mode = "--hairpin-mode=none" -%}
{% elif pillar['hairpin_mode'] is defined and pillar['hairpin_mode'] in ['promiscuous-bridge', 'hairpin-veth', 'none'] -%}
  {% set hairpin_mode = "--hairpin-mode=" + pillar['hairpin_mode'] -%}
{% endif -%}

{% set kubelet_port = "" -%}
{% if pillar['kubelet_port'] is defined -%}
  {% set kubelet_port="--port=" + pillar['kubelet_port'] %}
{% endif -%}

{% set log_level = pillar['log_level'] -%}
{% if pillar['kubelet_test_log_level'] is defined -%}
  {% set log_level = pillar['kubelet_test_log_level'] -%}
{% endif -%}

{% set enable_custom_metrics = "" -%}
{% if pillar['enable_custom_metrics'] is defined -%}
  {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %}
{% endif -%}

{% set kube_proxy_ds_label = "" %}
{% if grains['roles'][0] != 'kubernetes-master' and pillar.get('kube_proxy_daemonset', '').lower() == 'true' %}
  # Add kube-proxy daemonset label to node to avoid situation during cluster
  # upgrade/downgrade when there are two instances of kube-proxy running on a node.
  {% set kube_proxy_ds_label = "beta.kubernetes.io/kube-proxy-ds-ready=true," %}
{% endif %}
{% set node_labels = kube_proxy_ds_label + pillar['node_labels'] %}
{% if node_labels != "" %}
  {% set node_labels="--node-labels=" + node_labels %}
{% endif %}

{% set node_taints = "" %}
{% if pillar['node_taints'] is defined -%}
  {% set node_taints="--register-with-taints=" + pillar['node_taints'] %}
{% endif -%}

{% set eviction_hard = "" %}
{% if pillar['eviction_hard'] is defined -%}
  {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %}
{% endif -%}

{% set kubelet_auth = "--anonymous-auth=false --authorization-mode=Webhook --client-ca-file=" + pillar.get('ca_cert_bundle_path', '/var/lib/kubelet/ca.crt') %}
{% set pki=" --cert-dir=/var/lib/kubelet/pki" -%}

# test_args has to be kept at the end, so they'll overwrite any prior configuration
DAEMON_ARGS="{{daemon_args}} {{bootstrap_kubeconfig}} {{kubeconfig}} {{require_kubeconfig}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}}  {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{node_taints}} {{eviction_hard}} {{kubelet_auth}} {{pki}} {{feature_gates}} {{test_args}}"