github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement Azure cloud provider scripts

Browse Source 
Implement basic cloud provider functionality to deploy Kubernetes on
Azure.  SaltStack is used to deploy Kubernetes on top of Ubuntu
virtual machines.  OpenVpn provides network connectivity.  For
kubelet authentication, we use basic authentication (username and
password).  The scripts use the legacy Azure Service Management APIs.

We have set up a nightly test job in our Jenkins server for federated
testing to run the e2e test suite on Azure.  With the cloud provider
scripts in this commit, 14 e2e test cases pass in this environment.
We plan to implement additional Azure functionality to support more
test cases.
This commit is contained in:
Weixu Zhuang
2016-01-26 13:46:02 -08:00
committed by weixu
parent 4523429b20
commit e35c1ccba2
22 changed files with 542 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cluster/saltbase/README.md
View File

@@ -8,7 +8,7 @@ configurations for Debian-on-GCE, Fedora-on-Vagrant, Ubuntu-on-AWS and
Ubuntu-on-Azure. (That doesn't mean it can't be made to apply to an
arbitrary configuration, but those are only the in-tree OS/IaaS
combinations supported today.) As you peruse the configuration, these
are shorthanded as `gce`, `vagrant`, `aws`, `azure` in `grains.cloud`;
are shorthanded as `gce`, `vagrant`, `aws`, `azure-legacy` in `grains.cloud`;
the documentation in this tree uses this same shorthand for convenience.
See more:

88
cluster/saltbase/salt/docker/init.sls
View File

@@ -47,6 +47,93 @@ docker:
- pkg: docker-io
{% endif %}
{% elif grains.cloud is defined and grains.cloud == 'azure-legacy' %}
{% if pillar.get('is_systemd') %}
{{ pillar.get('systemd_system_path') }}/docker.service:
file.managed:
- source: salt://docker/docker.service
- template: jinja
- user: root
- group: root
- mode: 644
- defaults:
environment_file: {{ environment_file }}
# The docker service.running block below doesn't work reliably
# Instead we run our script which e.g. does a systemd daemon-reload
# But we keep the service block below, so it can be used by dependencies
# TODO: Fix this
fix-service-docker:
cmd.wait:
- name: /opt/kubernetes/helpers/services bounce docker
- watch:
- file: {{ pillar.get('systemd_system_path') }}/docker.service
- file: {{ environment_file }}
{% endif %}
{{ environment_file }}:
file.managed:
- source: salt://docker/docker-defaults
- template: jinja
- user: root
- group: root
- mode: 644
- makedirs: true
- require:
- pkg: docker-engine
apt-key:
pkgrepo.managed:
- humanname: Dotdeb
- name: deb https://apt.dockerproject.org/repo ubuntu-trusty main
- dist: ubuntu-trusty
- file: /etc/apt/sources.list.d/docker.list
- keyid: 58118E89F3A912897C070ADBF76221572C52609D
- keyserver: hkp://p80.pool.sks-keyservers.net:80
lxc-docker:
pkg:
- purged
docker-io:
pkg:
- purged
cbr0:
network.managed:
- enabled: True
- type: bridge
{% if grains['roles'][0] == 'kubernetes-pool' %}
- proto: none
{% else %}
- proto: dhcp
{% endif %}
- ports: none
- bridge: cbr0
{% if grains['roles'][0] == 'kubernetes-pool' %}
- ipaddr: {{ grains['cbr-cidr'] }}
{% endif %}
- delay: 0
- bypassfirewall: True
- require_in:
- service: docker
docker-engine:
pkg:
- installed
- require:
- pkgrepo: 'apt-key'
docker:
service.running:
- enable: True
- require:
- file: {{ environment_file }}
- watch:
- file: {{ environment_file }}
{% elif grains.cloud is defined and grains.cloud in ['vsphere', 'photon-controller'] and grains.os == 'Debian' and grains.osrelease_info[0] >=8 %}
{% if pillar.get('is_systemd') %}
@@ -304,6 +391,7 @@ docker-upgrade:
- name: /opt/kubernetes/helpers/pkg install-no-start {{ docker_pkg_name }} {{ override_docker_ver }} /var/cache/docker-install/{{ override_deb }}
- require:
- file: /var/cache/docker-install/{{ override_deb }}
{% endif %} # end override_docker_ver != ''
{% if pillar.get('is_systemd') %}

2
cluster/saltbase/salt/generate-cert/init.sls
View File

@@ -6,7 +6,7 @@
{% if grains.cloud == 'aws' %}
{% set cert_ip='_use_aws_external_ip_' %}
{% endif %}
{% if grains.cloud == 'azure' %}
{% if grains.cloud == 'azure-legacy' %}
{% set cert_ip='_use_azure_dns_name_' %}
{% endif %}
{% if grains.cloud == 'vsphere' or grains.cloud == 'photon-controller' %}

2
cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
View File

@@ -14,7 +14,7 @@
{% set srv_sshproxy_path = "/srv/sshproxy" -%}
{% if grains.cloud is defined -%}
{% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller'] -%}
{% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller', 'azure-legacy'] -%}
{% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
{% endif -%}

4
cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest
View File

@@ -36,7 +36,7 @@
{% set srv_kube_path = "/srv/kubernetes" -%}
{% if grains.cloud is defined -%}
{% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller'] -%}
{% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller', 'azure-legacy'] -%}
{% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
{% endif -%}
{% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%}
@@ -54,7 +54,7 @@
{% set root_ca_file = "" -%}
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %}
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack', 'azure-legacy'] %}
{% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%}
{% endif -%}

6
cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh
View File

@@ -24,7 +24,7 @@ function load-docker-images() {
if which docker 1>/dev/null 2>&1; then
timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1
timeout 120 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1
rc=$?
if [[ $rc == 0 ]]; then
let loadedImageFlags="$loadedImageFlags|1"
@@ -32,7 +32,7 @@ function load-docker-images() {
restart_docker=true
fi
timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1
timeout 120 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1
rc=$?
if [[ $rc == 0 ]]; then
let loadedImageFlags="$loadedImageFlags|2"
@@ -40,7 +40,7 @@ function load-docker-images() {
restart_docker=true
fi
timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1
timeout 120 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1
rc=$?
if [[ $rc == 0 ]]; then
let loadedImageFlags="$loadedImageFlags|4"

2
cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh
View File

@@ -22,7 +22,7 @@ while true; do
if which docker 1>/dev/null 2>&1; then
timeout 30 docker load -i /srv/salt/kube-bins/kube-proxy.tar 1>/dev/null 2>&1
timeout 120 docker load -i /srv/salt/kube-bins/kube-proxy.tar 1>/dev/null 2>&1
rc=$?
if [[ "${rc}" == 0 ]]; then
let loadedImageFlags="${loadedImageFlags}|1"

35
cluster/saltbase/salt/kube-proxy/default
View File

@@ -1,35 +0,0 @@
{% set daemon_args = "$DAEMON_ARGS" -%}
{% if grains['os_family'] == 'RedHat' -%}
{% set daemon_args = "" -%}
{% endif -%}
{# TODO(azure-maintainer): add support for distributing kubeconfig with token to kube-proxy #}
{# so it can use https #}
{% if grains['cloud'] is defined and grains['cloud'] == 'azure' -%}
{% set api_servers = "--master=http://" + ips[0][0] -%}
{% set api_servers_with_port = api_servers + ":7080" -%}
{% set kubeconfig = "" -%}
{% else -%}
{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
{% if grains.api_servers is defined -%}
{% set api_servers = "--master=https://" + grains.api_servers -%}
{% else -%}
{% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
{% set api_servers = "--master=https://" + ips[0][0] -%}
{% endif -%}
# TODO: remove nginx for other cloud providers.
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ] %}
{% set api_servers_with_port = api_servers -%}
{% else -%}
{% set api_servers_with_port = api_servers + ":6443" -%}
{% endif -%}
{% endif -%}
{% set test_args = "" -%}
{% if pillar['kubeproxy_test_args'] is defined -%}
{% set test_args=pillar['kubeproxy_test_args'] %}
{% endif -%}
# test_args has to be kept at the end, so they'll overwrite any prior configuration
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{kubeconfig}} {{pillar['log_level']}} {{test_args}}"

2
cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
View File

@@ -5,7 +5,7 @@
{% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
{% set api_servers = "--master=https://" + ips[0][0] -%}
{% endif -%}
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack' ] %}
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack', 'azure-legacy' ] %}
{% set api_servers_with_port = api_servers -%}
{% else -%}
{% set api_servers_with_port = api_servers + ":6443" -%}

6
cluster/saltbase/salt/kubelet/default
View File

@@ -16,7 +16,7 @@
{% endif -%}
# TODO: remove nginx for other cloud providers.
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %}
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack', 'azure-legacy'] %}
{% set api_servers_with_port = api_servers -%}
{% else -%}
{% set api_servers_with_port = api_servers + ":6443" -%}
@@ -28,7 +28,7 @@
{% set reconcile_cidr_args = "" -%}
{% if grains['roles'][0] == 'kubernetes-master' -%}
{% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] -%}
{% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack', 'azure-legacy'] -%}
# Unless given a specific directive, disable registration for the kubelet
# running on the master.
@@ -48,7 +48,7 @@
{% endif -%}
{% set cloud_provider = "" -%}
{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'vsphere', 'photon-controller'] -%}
{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'vsphere', 'photon-controller', 'azure-legacy'] -%}
{% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
{% endif -%}

64
cluster/saltbase/salt/nginx/init.sls Normal file
View File

@@ -0,0 +1,64 @@
nginx:
pkg:
- installed
/etc/nginx/nginx.conf:
file:
- managed
- source: salt://nginx/nginx.conf
- template: jinja
- user: root
- group: root
- mode: 644
/etc/nginx/sites-enabled/default:
file:
- managed
- makedirs: true
- source: salt://nginx/kubernetes-site
- user: root
- group: root
- mode: 644
/usr/share/nginx/htpasswd:
file:
- managed
- source: salt://nginx/htpasswd
- user: root
- group: root
- mode: 644
{% if grains.cloud is defined and grains.cloud in ['gce'] %}
/etc/kubernetes/manifests/nginx.json:
file:
- managed
- source: salt://nginx/nginx.json
- user: root
- group: root
- mode: 644
- require:
- file: /etc/nginx/nginx.conf
- file: /etc/nginx/sites-enabled/default
- file: /usr/share/nginx/htpasswd
- cmd: kubernetes-cert
#stop legacy nginx_service
stop_nginx-service:
service.dead:
- name: nginx
- enable: None
{% else %}
nginx-service:
service:
- running
- name: nginx
- watch:
- pkg: nginx
- file: /etc/nginx/nginx.conf
- file: /etc/nginx/sites-enabled/default
- file: /usr/share/nginx/htpasswd
- cmd: kubernetes-cert
{% endif %}

66
cluster/saltbase/salt/nginx/kubernetes-site Normal file
View File

@@ -0,0 +1,66 @@
#server {
#listen 80; ## listen for ipv4; this line is default and implied
#listen [::]:80 default_server ipv6only=on; ## listen for ipv6
# root /usr/share/nginx/www;
# index index.html index.htm;
# Make site accessible from http://localhost/
# server_name localhost;
# location / {
# auth_basic "Restricted";
# auth_basic_user_file /usr/share/nginx/htpasswd;
# Proxy settings.
# proxy_pass http://localhost:8080/;
# proxy_connect_timeout 159s;
# proxy_send_timeout 600s;
# proxy_read_timeout 600s;
# proxy_buffer_size 64k;
# proxy_buffers 16 32k;
# proxy_busy_buffers_size 64k;
# proxy_temp_file_write_size 64k;
# }
#}
# HTTPS server
#
server {
listen 443;
server_name localhost;
root html;
index index.html index.htm;
ssl on;
ssl_certificate /srv/kubernetes/server.cert;
ssl_certificate_key /srv/kubernetes/server.key;
ssl_session_timeout 5m;
# don't use SSLv3 because of POODLE
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS;
ssl_prefer_server_ciphers on;
location / {
auth_basic "Restricted";
auth_basic_user_file /usr/share/nginx/htpasswd;
# Proxy settings
# disable buffering so that watch works
proxy_buffering off;
proxy_pass http://127.0.0.1:8080/;
proxy_connect_timeout 159s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
# Disable retry
proxy_next_upstream off;
# Support web sockets
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}

61
cluster/saltbase/salt/nginx/nginx.conf Normal file
View File

@@ -0,0 +1,61 @@
{% if grains['os_family'] == 'RedHat' %}
user nginx;
{% else %}
user www-data;
{% endif %}
worker_processes 4;
pid /var/run/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}

60
cluster/saltbase/salt/nginx/nginx.json Normal file
View File

@@ -0,0 +1,60 @@
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {"name":"nginx"},
"spec":{
"hostNetwork": true,
"containers":[
{
"name": "nginx",
"image": "gcr.io/google-containers/nginx:v1",
"resources": {
"limits": {
"cpu": "200m"
}
},
"command": [
"nginx",
"-g",
"daemon off;"
],
"ports":[
{ "name": "https",
"containerPort": 443,
"hostPort": 443}
],
"volumeMounts": [
{ "name": "nginx",
"mountPath": "/etc/nginx",
"readOnly": true},
{ "name": "k8s",
"mountPath": "/srv/kubernetes",
"readOnly": true},
{ "name": "logs",
"mountPath": "/var/log/nginx",
"readOnly": false},
{ "name": "passwd",
"mountPath": "/usr/share/nginx",
"readOnly": true}
]
}
],
"volumes":[
{ "name": "nginx",
"hostPath": {
"path": "/etc/nginx"}
},
{ "name": "k8s",
"hostPath": {
"path": "/srv/kubernetes"}
},
{ "name": "passwd",
"hostPath": {
"path": "/usr/share/nginx"}
},
{ "name": "logs",
"hostPath": {
"path": "/var/logs/nginx"}
}
]
}}

6
cluster/saltbase/salt/openvpn/init.sls
View File

@@ -7,10 +7,10 @@
- mode: 644
- makedirs: True
{% for (minion, grains) in salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').items() %}
/etc/openvpn/ccd/{{ minion }}:
{% for minion in salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values() %}
/etc/openvpn/ccd/{{ minion['hostnamef'] }}:
file.managed:
- contents: "iroute {{ grains['cbr-string'] }}\n"
- contents: "iroute {{ minion['cbr-string'] }}\n"
- user: root
- group: root
- mode: 644

7
cluster/saltbase/salt/top.sls
View File

@@ -20,7 +20,7 @@ base:
{% elif pillar.get('network_provider', '').lower() == 'cni' %}
- cni
{% endif %}
{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %}
{% if grains['cloud'] is defined and grains['cloud'] == 'azure-legacy' %}
- openvpn-client
{% endif %}
- helpers
@@ -81,10 +81,11 @@ base:
- logrotate
{% endif %}
- kube-addons
{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %}
{% if grains['cloud'] is defined and grains['cloud'] == 'azure-legacy' %}
- openvpn
- nginx
{% endif %}
{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller', 'openstack'] %}
{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller', 'openstack', 'azure-legacy'] %}
- docker
- kubelet
{% endif %}