diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79b834b87..0d45b4ea3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: uses: actions/checkout@v2 with: path: src/github.com/containerd/containerd - fetch-depth: 25 + fetch-depth: 100 - name: Checkout project repo uses: actions/checkout@v2 @@ -114,7 +114,7 @@ jobs: if [ -z "${GITHUB_COMMIT_URL}" ]; then DCO_RANGE=$(jq -r '.after + "..HEAD"' ${GITHUB_EVENT_PATH}) else - DCO_RANGE=$(curl ${GITHUB_COMMIT_URL} | jq -r '.[0].parents[0].sha +".."+ .[-1].sha') + DCO_RANGE=$(curl ${GITHUB_COMMIT_URL} | jq -r '.[0].parents[0].sha + "..HEAD"') fi ../project/script/validate/dco diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 327b44e5b..6c2f933dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,11 +66,18 @@ jobs: - name: Set env shell: bash + env: + MOS: ${{ matrix.os }} run: | releasever=${{ github.ref }} releasever="${releasever#refs/tags/}" + os=linux + [[ "${MOS}" =~ "windows" ]] && { + os=windows + } echo "::set-env name=RELEASE_VER::${releasever}" echo "::set-env name=GOPATH::${{ github.workspace }}" + echo "::set-env name=OS::${os}" echo "::add-path::${{ github.workspace }}/bin" - name: Checkout containerd @@ -79,12 +86,14 @@ jobs: repository: containerd/containerd ref: ${{ github.ref }} path: src/github.com/containerd/containerd + - name: HCS Shim commit id: hcsshim_commit if: startsWith(matrix.os, 'windows') shell: bash run: echo "::set-output name=sha::$(grep Microsoft/hcsshim vendor.conf | awk '{print $2}')" working-directory: src/github.com/containerd/containerd + - name: Checkout hcsshim source if: startsWith(matrix.os, 'windows') uses: actions/checkout@v2 @@ -92,16 +101,14 @@ jobs: repository: Microsoft/hcsshim ref: ${{ steps.hcsshim_commit.outputs.sha }} path: src/github.com/Microsoft/hcsshim + - name: Make shell: bash - env: - MOS: ${{ matrix.os }} - OS: linux run: | make build make binaries - [[ "${MOS}" =~ "windows" ]] && { - OS=windows + rm bin/containerd-stress* + [[ "${OS}" == "windows" ]] && { ( bindir="$(pwd)/bin" cd ../../Microsoft/hcsshim @@ -119,6 +126,47 @@ jobs: name: containerd-binaries-${{ matrix.os }} path: src/github.com/containerd/containerd/*.tar.gz* + - name: Install cri-containerd dependencies + shell: bash + env: + RUNC_FLAVOR: runc + DESTDIR: ${{ github.workspace }}/cri-release + run: | + mkdir ${DESTDIR} + if [[ "${OS}" == "linux" ]]; then + sudo install -d ${DESTDIR}/usr/local/bin + sudo install -D -m 755 bin/* ${DESTDIR}/usr/local/bin + sudo install -d ${DESTDIR}/opt/containerd/cluster + sudo cp -r contrib/gce ${DESTDIR}/opt/containerd/cluster/ + sudo install -d ${DESTDIR}/etc/systemd/system + sudo install -m 644 containerd.service ${DESTDIR}/etc/systemd/system + echo "CONTAINERD_VERSION: '${RELEASE_VER#v}'" | sudo tee ${DESTDIR}/opt/containerd/cluster/version + + sudo PATH=$PATH script/setup/install-seccomp + USESUDO=true script/setup/install-runc + script/setup/install-cni + script/setup/install-critools + elif [[ "${OS}" == "windows" ]]; then + script/setup/install-cni-windows + cp bin/* ${DESTDIR}/ + fi + working-directory: src/github.com/containerd/containerd + + - name: Make cri-containerd tar + shell: bash + run: | + TARFILE="cri-containerd-cni-${RELEASE_VER#v}-${OS}-amd64.tar.gz" + [[ "${OS}" == "linux" ]] && tar czf ${TARFILE} etc usr opt + [[ "${OS}" == "windows" ]] && tar czf ${TARFILE} * + sha256sum ${TARFILE} >${TARFILE}.sha256sum + working-directory: cri-release + + - name: Save cri-containerd binaries + uses: actions/upload-artifact@v2 + with: + name: cri-containerd-binaries-${{ matrix.os }} + path: cri-release/cri-containerd-cni-*.tar.gz* + release: name: Create containerd Release runs-on: ubuntu-18.04 @@ -135,8 +183,12 @@ jobs: run: | _filenum=1 for i in "ubuntu-18.04" "windows-2019"; do - for i in `ls builds/containerd-binaries-${i}`; do - echo "::set-output name=file${_filenum}::${i}" + for f in `ls builds/containerd-binaries-${i}`; do + echo "::set-output name=file${_filenum}::${f}" + let "_filenum+=1" + done + for f in `ls builds/cri-containerd-binaries-${i}`; do + echo "::set-output name=file${_filenum}::${f}" let "_filenum+=1" done done @@ -169,14 +221,32 @@ jobs: asset_path: ./builds/containerd-binaries-ubuntu-18.04/${{ steps.catalog.outputs.file2 }} asset_name: ${{ steps.catalog.outputs.file2 }} asset_content_type: text/plain + - name: Upload Linux cri containerd tarball + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-ubuntu-18.04/${{ steps.catalog.outputs.file3 }} + asset_name: ${{ steps.catalog.outputs.file3 }} + asset_content_type: application/gzip + - name: Upload Linux cri sha256 sum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-ubuntu-18.04/${{ steps.catalog.outputs.file4 }} + asset_name: ${{ steps.catalog.outputs.file4 }} + asset_content_type: text/plain - name: Upload Windows containerd tarball uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file3 }} - asset_name: ${{ steps.catalog.outputs.file3 }} + asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file5 }} + asset_name: ${{ steps.catalog.outputs.file5 }} asset_content_type: application/gzip - name: Upload Windows sha256 sum uses: actions/upload-release-asset@v1 @@ -184,6 +254,24 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file4 }} - asset_name: ${{ steps.catalog.outputs.file4 }} + asset_path: ./builds/containerd-binaries-windows-2019/${{ steps.catalog.outputs.file6 }} + asset_name: ${{ steps.catalog.outputs.file6 }} + asset_content_type: text/plain + - name: Upload Windows cri containerd tarball + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-windows-2019/${{ steps.catalog.outputs.file7 }} + asset_name: ${{ steps.catalog.outputs.file7 }} + asset_content_type: application/gzip + - name: Upload Windows cri sha256 sum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./builds/cri-containerd-binaries-windows-2019/${{ steps.catalog.outputs.file8 }} + asset_name: ${{ steps.catalog.outputs.file8 }} asset_content_type: text/plain diff --git a/containerd.service b/containerd.service index cae30ec7c..5f67110ab 100644 --- a/containerd.service +++ b/containerd.service @@ -11,6 +11,7 @@ Type=notify Delegate=yes KillMode=process Restart=always +RestartSec=5 # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNPROC=infinity @@ -19,6 +20,7 @@ LimitNOFILE=1048576 # Comment TasksMax if your systemd version does not supports it. # Only systemd 226 and above support this version. TasksMax=infinity +OOMScoreAdjust=-999 [Install] WantedBy=multi-user.target diff --git a/contrib/gce/cloud-init/master.yaml b/contrib/gce/cloud-init/master.yaml new file mode 100644 index 000000000..35c869c5b --- /dev/null +++ b/contrib/gce/cloud-init/master.yaml @@ -0,0 +1,199 @@ +#cloud-config + +users: +- name: etcd + homedir: /var/etcd + lock_passwd: true + ssh_redirect_user: true + +write_files: +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Download and install containerd binaries and configurations. + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh + + [Install] + WantedBy=containerd.target + + - path: /etc/systemd/system/containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + OOMScoreAdjust=-999 + LimitNOFILE=1048576 + # Having non-zero Limit*s causes performance problems due to accounting overhead + # in the kernel. We recommend using cgroups to do container-local accounting. + LimitNPROC=infinity + LimitCORE=infinity + TasksMax=infinity + ExecStartPre=/sbin/modprobe overlay + ExecStart=/home/containerd/usr/local/bin/containerd + + [Install] + WantedBy=containerd.target + + - path: /etc/systemd/system/containerd.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Containerd + + [Install] + WantedBy=kubernetes.target + +# Setup kubernetes. + - path: /etc/systemd/system/kube-master-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-master-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-container-runtime-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for container runtime + After=kube-master-configuration.service + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-master-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + +runcmd: + # Stop the existing containerd service if there is one. (for Docker 18.09+) + - systemctl is-active containerd && systemctl stop containerd + - systemctl daemon-reload + - systemctl enable containerd-installation.service + - systemctl enable containerd.service + - systemctl enable containerd.target + - systemctl enable kube-master-installation.service + - systemctl enable kube-master-configuration.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-container-runtime-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target + # Start docker after containerd is running. (for Docker 18.09+) + - systemctl is-enabled docker && (systemctl is-active docker || systemctl start docker) diff --git a/contrib/gce/cloud-init/node.yaml b/contrib/gce/cloud-init/node.yaml new file mode 100644 index 000000000..97d7ad429 --- /dev/null +++ b/contrib/gce/cloud-init/node.yaml @@ -0,0 +1,193 @@ +#cloud-config + +write_files: +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=Download and install containerd binaries and configurations. + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh + + [Install] + WantedBy=containerd.target + + - path: /etc/systemd/system/containerd.service + permissions: 0644 + owner: root + content: | + # installed by cloud-init + [Unit] + Description=containerd container runtime + Documentation=https://containerd.io + After=containerd-installation.service + + [Service] + Restart=always + RestartSec=5 + Delegate=yes + KillMode=process + OOMScoreAdjust=-999 + LimitNOFILE=1048576 + # Having non-zero Limit*s causes performance problems due to accounting overhead + # in the kernel. We recommend using cgroups to do container-local accounting. + LimitNPROC=infinity + LimitCORE=infinity + TasksMax=infinity + ExecStartPre=/sbin/modprobe overlay + ExecStart=/home/containerd/usr/local/bin/containerd + + [Install] + WantedBy=containerd.target + + - path: /etc/systemd/system/containerd.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Containerd + + [Install] + WantedBy=kubernetes.target + +# Setup kubernetes. + - path: /etc/systemd/system/kube-node-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-node-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes node + After=kube-node-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-container-runtime-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for container runtime + After=kube-node-configuration.service + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-node-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + +runcmd: + # Stop the existing containerd service if there is one. (for Docker 18.09+) + - systemctl is-active containerd && systemctl stop containerd + - systemctl daemon-reload + - systemctl enable containerd-installation.service + - systemctl enable containerd.service + - systemctl enable containerd.target + - systemctl enable kube-node-installation.service + - systemctl enable kube-node-configuration.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-container-runtime-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target + # Start docker after containerd is running. (for Docker 18.09+) + - systemctl is-enabled docker && (systemctl is-active docker || systemctl start docker) diff --git a/contrib/gce/cni.template b/contrib/gce/cni.template new file mode 100644 index 000000000..c8c0d7f52 --- /dev/null +++ b/contrib/gce/cni.template @@ -0,0 +1,21 @@ +{ + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "ranges": [{{range $i, $range := .PodCIDRRanges}}{{if $i}}, {{end}}[{"subnet": "{{$range}}"}]{{end}}], + "routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] +} diff --git a/contrib/gce/configure.sh b/contrib/gce/configure.sh new file mode 100755 index 000000000..c2a05bbba --- /dev/null +++ b/contrib/gce/configure.sh @@ -0,0 +1,218 @@ +#!/bin/bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +# CONTAINERD_HOME is the directory for containerd. +CONTAINERD_HOME="/home/containerd" +cd "${CONTAINERD_HOME}" +# KUBE_HOME is the directory for kubernetes. +KUBE_HOME="/home/kubernetes" + +# fetch_metadata fetches metadata from GCE metadata server. +# Var set: +# 1. Metadata key: key of the metadata. +fetch_metadata() { + local -r key=$1 + local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes" + if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \ + grep -q "^${key}$"; then + curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \ + "${attributes}/${key}" + fi +} + +# fetch_env fetches environment variables from GCE metadata server +# and generate a env file under ${CONTAINERD_HOME}. It assumes that +# the environment variables in metadata are in yaml format. +fetch_env() { + local -r env_file_name=$1 + ( + umask 077; + local -r tmp_env_file="/tmp/${env_file_name}.yaml" + tmp_env_content=$(fetch_metadata "${env_file_name}") + if [ -z "${tmp_env_content}" ]; then + echo "No environment variable is specified in ${env_file_name}" + return + fi + echo "${tmp_env_content}" > "${tmp_env_file}" + # Convert the yaml format file into a shell-style file. + eval $(python -c ''' +import pipes,sys,yaml +for k,v in yaml.load(sys.stdin).iteritems(): + print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))) +''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}") + rm -f "${tmp_env_file}" + ) +} + +# is_preloaded checks whether a package has been preloaded in the image. +is_preloaded() { + local -r tar=$1 + local -r sha1=$2 + grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info" +} + +# KUBE_ENV_METADATA is the metadata key for kubernetes envs. +KUBE_ENV_METADATA="kube-env" +fetch_env ${KUBE_ENV_METADATA} +if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then + source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" +fi + +# CONTAINERD_ENV_METADATA is the metadata key for containerd envs. +CONTAINERD_ENV_METADATA="containerd-env" +fetch_env ${CONTAINERD_ENV_METADATA} +if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then + source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" +fi + +# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name. +# By default use the release tarball with cni built in. +pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"} +# Behave differently for test and production. +if [ "${CONTAINERD_TEST:-"false"}" != "true" ]; then + # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored. + deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"} + # CONTAINERD_VERSION is the cri-containerd version to use. + version=${CONTAINERD_VERSION:-""} +else + deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"} + + # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow. + PULL_REFS_METADATA="PULL_REFS" + pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}") + if [ ! -z "${pull_refs}" ]; then + deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}') + deploy_path="${deploy_path}/${deploy_dir}" + fi + + # TODO(random-liu): Put version into the metadata instead of + # deciding it in cloud init. This may cause issue to reboot test. + version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \ + https://storage.googleapis.com/${deploy_path}/latest) +fi + +TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz" +# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e. +TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}" +# TARBALL is the name of the tarball after being downloaded. +TARBALL="cri-containerd.tar.gz" +# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball. +tar_sha1="${CONTAINERD_TAR_SHA1:-""}" + +if [ -z "${version}" ]; then + # Try using preloaded containerd if version is not specified. + tarball_gcs_pattern="${pkg_prefix}-.*.linux-amd64.tar.gz" + if is_preloaded "${tarball_gcs_pattern}" "${tar_sha1}"; then + echo "CONTAINERD_VERSION is not set, use preloaded containerd" + else + echo "CONTAINERD_VERSION is not set, and containerd is not preloaded" + exit 1 + fi +else + if is_preloaded "${TARBALL_GCS_NAME}" "${tar_sha1}"; then + echo "${TARBALL_GCS_NAME} is preloaded" + else + # Download and untar the release tar ball. + curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}" + tar xvf "${TARBALL}" + rm -f "${TARBALL}" + fi +fi + +# Remove crictl shipped with containerd, use crictl installed +# by kube-up.sh. +rm -f "${CONTAINERD_HOME}/usr/local/bin/crictl" +rm -f "${CONTAINERD_HOME}/etc/crictl.yaml" + +# Generate containerd config +config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}" +mkdir -p $(dirname ${config_path}) +cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin" +cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template" +if [ "${KUBERNETES_MASTER:-}" != "true" ]; then + if [ "${NETWORK_POLICY_PROVIDER:-"none"}" != "none" ] || [ "${ENABLE_NETD:-}" == "true" ]; then + # Use Kubernetes cni daemonset on node if network policy provider is specified + # or netd is enabled. + cni_bin_dir="${KUBE_HOME}/bin" + cni_template_path="" + fi +fi +log_level="${CONTAINERD_LOG_LEVEL:-"info"}" +max_container_log_line="${CONTAINERD_MAX_CONTAINER_LOG_LINE:-16384}" +cat > ${config_path} <> ${config_path} < \ + /etc/profile.d/containerd_env.sh + +# Run extra init script for test. +if [ "${CONTAINERD_TEST:-"false"}" == "true" ]; then + # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. + EXTRA_INIT_SCRIPT="containerd-extra-init.sh" + # EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script. + EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh" + extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}") + # Return if containerd-extra-init-sh is not set. + if [ -z "${extra_init}" ]; then + exit 0 + fi + echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}" + chmod 544 "${EXTRA_INIT_SCRIPT}" + ./${EXTRA_INIT_SCRIPT} +fi diff --git a/contrib/gce/env b/contrib/gce/env new file mode 100644 index 000000000..3e3bb01f9 --- /dev/null +++ b/contrib/gce/env @@ -0,0 +1,20 @@ +#!/bin/bash +GCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# TODO(random-liu): Upload release tarball to user's own GCS, and use it. We should +# not let all nodes of all users download tarball from cri-containerd-release. +version_file=${GCE_DIR}/../version +if [ ! -f "${version_file}" ]; then + echo "version file does not exist" + exit 1 +fi +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}" +export KUBE_CONTAINER_RUNTIME="remote" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="unix:///run/containerd/containerd.sock" +export KUBE_CONTAINER_RUNTIME_NAME=containerd +export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr -n=k8s.io images import" +export NETWORK_PROVIDER="" +export NON_MASQUERADE_CIDR="0.0.0.0/0" +export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service" +export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true" diff --git a/script/setup/install-cni b/script/setup/install-cni index 6d443da07..0d1e9b331 100755 --- a/script/setup/install-cni +++ b/script/setup/install-cni @@ -22,8 +22,8 @@ set -eu -o pipefail CNI_COMMIT=$(grep containernetworking/plugins "$GOPATH"/src/github.com/containerd/containerd/vendor.conf | awk '{print $2}') -CNI_DIR=/opt/cni -CNI_CONFIG_DIR=/etc/cni/net.d +CNI_DIR=${DESTDIR:=''}/opt/cni +CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d go get -d github.com/containernetworking/plugins/... cd "$GOPATH"/src/github.com/containernetworking/plugins @@ -32,7 +32,7 @@ FASTBUILD=true ./build.sh sudo mkdir -p $CNI_DIR sudo cp -r ./bin $CNI_DIR sudo mkdir -p $CNI_CONFIG_DIR -cat <= 8 )); then + mask=255 + elif (( len > 0 )); then + mask=$(( 256 - 2 ** ( 8 - len ) )) + else + mask=0 + fi + (( len -= 8 )) + result_array[i]=$(( gateway_array[i] & mask )) + done + result="$(printf ".%s" "${result_array[@]}")" + result="${result:1}" + echo "$result/$((32 - prefix_len))" +} + +# nat already exists on the Windows VM, the subnet and gateway +# we specify should match that. +gateway="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).IPAddress")" +prefix_len="$(powershell -c "(Get-NetIPAddress -InterfaceAlias 'vEthernet (nat)' -AddressFamily IPv4).PrefixLength")" + +subnet="$(calculate_subnet "$gateway" "$prefix_len")" + +# The "name" field in the config is used as the underlying +# network type right now (see +# https://github.com/microsoft/windows-container-networking/pull/45), +# so it must match a network type in: +# https://docs.microsoft.com/en-us/windows-server/networking/technologies/hcn/hcn-json-document-schemas +bash -c 'cat >"'"${CNI_CONFIG_DIR}"'"/0-containerd-nat.conf <