From f3c5d4e62bf4154ae080f5ce6d6eee12dde03681 Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Mon, 6 Jun 2016 10:58:57 -0700 Subject: [PATCH] cluster/gce/coreos: Update the cloud config and scripts to enable rkt runtime. --- cluster/gce/coreos/configure-node.sh | 2 +- cluster/gce/coreos/helper.sh | 4 +- .../coreos/kube-manifests/etcd-events.yaml | 2 +- .../kube-manifests/kube-addon-manager.yaml | 4 +- .../{master.yaml => master-docker.yaml} | 24 +- cluster/gce/coreos/master-rkt.yaml | 209 ++++++++++++++++++ cluster/gce/coreos/node-docker.yaml | 161 ++++++++++++++ .../gce/coreos/{node.yaml => node-rkt.yaml} | 19 +- 8 files changed, 384 insertions(+), 41 deletions(-) rename cluster/gce/coreos/{master.yaml => master-docker.yaml} (87%) create mode 100644 cluster/gce/coreos/master-rkt.yaml create mode 100644 cluster/gce/coreos/node-docker.yaml rename cluster/gce/coreos/{node.yaml => node-rkt.yaml} (93%) diff --git a/cluster/gce/coreos/configure-node.sh b/cluster/gce/coreos/configure-node.sh index c03a2b083da..7a9c0c65e51 100644 --- a/cluster/gce/coreos/configure-node.sh +++ b/cluster/gce/coreos/configure-node.sh @@ -119,7 +119,7 @@ function wait-url-up() { done } -# Configure addon yamls, and run salt/kube-addons/kube-addon.sh +# Configure addon yamls, and run salt/kube-addons/kube-addons.sh function configure-master-addons() { echo "Configuring master addons" diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 275f1e764fc..6e0fa89b298 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -23,7 +23,7 @@ function create-node-instance-template() { local template_name="$1" create-node-template "$template_name" "${scope_flags}" \ "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ - "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \ + "user-data=${KUBE_ROOT}/cluster/gce/coreos/node-${CONTAINER_RUNTIME}.yaml" \ "configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh" \ "configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh" \ "cluster-name=${KUBE_TEMP}/cluster-name.txt" @@ -65,7 +65,7 @@ function create-master-instance() { --scopes "storage-ro,compute-rw,monitoring,logging-write" \ --can-ip-forward \ --metadata-from-file \ - "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master-${CONTAINER_RUNTIME}.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \ --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ ${preemptible_master} diff --git a/cluster/gce/coreos/kube-manifests/etcd-events.yaml b/cluster/gce/coreos/kube-manifests/etcd-events.yaml index d6a92f0c62e..b682ec9aa65 100644 --- a/cluster/gce/coreos/kube-manifests/etcd-events.yaml +++ b/cluster/gce/coreos/kube-manifests/etcd-events.yaml @@ -14,7 +14,7 @@ spec: --bind-addr=127.0.0.1:4002 --data-dir=/var/etcd/data-events 1>>/var/log/etcd-events.log 2>&1 - image: gcr.io/google_containers/etcd:2.0.12 + image: gcr.io/google_containers/etcd:2.2.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: diff --git a/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml b/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml index d46b9888117..e63f23b5f38 100644 --- a/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml +++ b/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml @@ -12,7 +12,9 @@ spec: resources: requests: cpu: 5m - memory: 50Mi + # TODO(yifan): Figure out what's the memory usage should be here. + # See https://github.com/kubernetes/kubernetes/issues/23641. + memory: 100Mi volumeMounts: - mountPath: /etc/kubernetes/ name: addons diff --git a/cluster/gce/coreos/master.yaml b/cluster/gce/coreos/master-docker.yaml similarity index 87% rename from cluster/gce/coreos/master.yaml rename to cluster/gce/coreos/master-docker.yaml index 11a83fd094f..50849eaa5aa 100644 --- a/cluster/gce/coreos/master.yaml +++ b/cluster/gce/coreos/master-docker.yaml @@ -38,24 +38,6 @@ coreos: ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ - - name: kubernetes-install-rkt.service - command: start - content: | - [Unit] - Description=Fetch rkt - Documentation=http://github.com/coreos/rkt - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/ - ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz - ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite - - name: kubernetes-download-salt.service command: start content: | @@ -139,7 +121,7 @@ coreos: --cluster-dns=${DNS_SERVER_IP} \ --cluster-domain=${DNS_DOMAIN} \ --logtostderr=true \ - --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \ + --container-runtime=docker \ --rkt-path=/opt/rkt/rkt \ --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \ --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ @@ -149,7 +131,7 @@ coreos: Restart=always RestartSec=10 KillMode=process - + - name: docker.service drop-ins: - name: 50-docker-opts.conf @@ -171,8 +153,6 @@ coreos: Description=Configure Node For Kubernetes service Requires=kubernetes-install-node.service After=kubernetes-install-node.service - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service Requires=kubernetes-download-salt.service After=kubernetes-download-salt.service Requires=kubernetes-download-manifests.service diff --git a/cluster/gce/coreos/master-rkt.yaml b/cluster/gce/coreos/master-rkt.yaml new file mode 100644 index 00000000000..ede9fff5cc9 --- /dev/null +++ b/cluster/gce/coreos/master-rkt.yaml @@ -0,0 +1,209 @@ +#cloud-config + +coreos: + update: + reboot-strategy: off + units: + - name: locksmithd.service + mask: true + - name: kube-env.service + command: start + content: | + [Unit] + Description=Fetch kubernetes-node-environment + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /etc/kube-env.yaml \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + # Transform the yaml to env file. + ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env + ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env + + - name: kubernetes-install-cni.service + command: start + content: | + [Unit] + Description=Download cni + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/mkdir -p /opt/cni + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz + ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ + + - name: kubernetes-install-docker2aci.service + command: start + content: | + [Unit] + Description=Download docker2aci + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/mkdir -p /opt/docker2aci + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/docker2aci.tar.gz https://github.com/appc/docker2aci/releases/download/v0.11.1/docker2aci-v0.11.1.tar.gz + ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/docker2aci.tar.gz -C /opt/docker2aci/ --overwrite + + - name: kubernetes-install-rkt.service + command: start + content: | + [Unit] + Description=Fetch rkt + Documentation=http://github.com/coreos/rkt + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/ + ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz + ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite + + - name: kubernetes-download-salt.service + command: start + content: | + [Unit] + Description=Download salt + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL} + # TODO(yifan): Check hash. + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite + + - name: kubernetes-download-manifests.service + command: start + content: | + [Unit] + Description=Download manifests + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} + # TODO(yifan): Check hash. + ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite + + - name: kubernetes-install-node.service + command: start + content: | + [Unit] + Description=Install Kubernetes Server + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} + # TODO(yifan): Check hash. + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite + + - name: rkt-api-service.service + command: start + content: | + [Unit] + Description=Start rkt API service as Daemon + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + [Service] + ExecStart=/opt/rkt/rkt api-service + Restart=always + RestartSec=10 + + - name: kubelet.service + command: start + content: | + [Unit] + Description=Run Kubelet service + Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-kubelet.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet + ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh + ExecStartPre=/run/configure-kubelet.sh + ExecStart=/opt/kubernetes/server/bin/kubelet \ + --api-servers=https://${INSTANCE_PREFIX}-master \ + --enable-debugging-handlers=false \ + --cloud-provider=gce \ + --config=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --v=2 \ + --cluster-dns=${DNS_SERVER_IP} \ + --cluster-domain=${DNS_DOMAIN} \ + --logtostderr=true \ + --container-runtime=rkt \ + --rkt-path=/opt/rkt/rkt \ + --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \ + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ + --pod-cidr=${MASTER_IP_RANGE} \ + --register-schedulable=false \ + --reconcile-cidr=false + Restart=always + RestartSec=10 + KillMode=process + + - name: docker.service + command: stop + + - name: kubernetes-configure-node.service + command: start + content: | + [Unit] + Description=Configure Node For Kubernetes service + Requires=kubernetes-install-node.service + After=kubernetes-install-node.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + Requires=kubernetes-download-salt.service + After=kubernetes-download-salt.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service + Requires=kubernetes-install-docker2aci.service + After=kubernetes-install-docker2aci.service + # Need the kubelet/docker running because we will use docker load for docker images. + Requires=kubelet.service + After=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-node.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node + ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh + ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/node-docker.yaml b/cluster/gce/coreos/node-docker.yaml new file mode 100644 index 00000000000..0246c056a10 --- /dev/null +++ b/cluster/gce/coreos/node-docker.yaml @@ -0,0 +1,161 @@ +#cloud-config + +coreos: + update: + reboot-strategy: off + units: + - name: locksmithd.service + mask: true + - name: kube-env.service + command: start + content: | + [Unit] + Description=Fetch kubernetes-node-environment + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /etc/kube-env.yaml \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + # Transform the yaml to env file. + ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env + ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env + + - name: kubernetes-install-cni.service + command: start + content: | + [Unit] + Description=Download cni + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/mkdir -p /opt/cni + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz + ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ + + - name: kubernetes-download-manifests.service + command: start + content: | + [Unit] + Description=Download manifests + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} + # TODO(yifan): Check hash. + ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite + + - name: kubernetes-install-node.service + command: start + content: | + [Unit] + Description=Install Kubernetes Server + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} + ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite + + - name: kubelet.service + command: start + content: | + [Unit] + Description=Run Kubelet service + Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-kubelet.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet + ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh + ExecStartPre=/run/configure-kubelet.sh + ExecStart=/opt/kubernetes/server/bin/kubelet \ + --api-servers=https://${INSTANCE_PREFIX}-master \ + --enable-debugging-handlers=true \ + --cloud-provider=gce \ + --config=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --v=2 \ + --cluster-dns=${DNS_SERVER_IP} \ + --cluster-domain=${DNS_DOMAIN} \ + --logtostderr=true \ + --container-runtime=docker \ + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ + --network-plugin=${NETWORK_PROVIDER} \ + --reconcile-cidr=true + Restart=always + RestartSec=10 + KillMode=process + + - name: kube-proxy.service + command: start + content: | + [Unit] + Description=Start Kube-proxy service as Daemon + Requires=kubernetes-configure-node.service + After=kubernetes-configure-node.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStart=/opt/kubernetes/server/bin/kube-proxy \ + --master=https://${KUBERNETES_MASTER_NAME} \ + --kubeconfig=/var/lib/kube-proxy/kubeconfig \ + --v=2 \ + --logtostderr=true + Restart=always + RestartSec=10 + + - name: docker.service + drop-ins: + - name: 50-docker-opts.conf + content: | + [Service] + Environment='DOCKER_OPTS= --iptables=false --ip-masq=false' + MountFlags=slave + LimitNOFILE=1048576 + LimitNPROC=1048576 + LimitCORE=infinity + Restart=always + RestartSec=2s + StartLimitInterval=0 + + - name: kubernetes-configure-node.service + command: start + content: | + [Unit] + Description=Configure Node For Kubernetes service + Requires=kubernetes-install-node.service + After=kubernetes-install-node.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-node.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node + ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh + ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/node.yaml b/cluster/gce/coreos/node-rkt.yaml similarity index 93% rename from cluster/gce/coreos/node.yaml rename to cluster/gce/coreos/node-rkt.yaml index 4d07b94c6aa..803ac735697 100644 --- a/cluster/gce/coreos/node.yaml +++ b/cluster/gce/coreos/node-rkt.yaml @@ -131,10 +131,12 @@ coreos: --cluster-dns=${DNS_SERVER_IP} \ --cluster-domain=${DNS_DOMAIN} \ --logtostderr=true \ - --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \ + --container-runtime=rkt \ --rkt-path=/opt/rkt/rkt \ --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \ - --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ + --network-plugin=kubenet \ + --reconcile-cidr=true Restart=always RestartSec=10 KillMode=process @@ -157,18 +159,7 @@ coreos: RestartSec=10 - name: docker.service - drop-ins: - - name: 50-docker-opts.conf - content: | - [Service] - Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false' - MountFlags=slave - LimitNOFILE=1048576 - LimitNPROC=1048576 - LimitCORE=infinity - Restart=always - RestartSec=2s - StartLimitInterval=0 + command: stop - name: kubernetes-configure-node.service command: start