From a69f3555da8a0b07157674b900657186c70ede14 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 16 Mar 2018 06:40:14 +0000 Subject: [PATCH] Add build-containerd.sh to build containerd from existing repo Signed-off-by: Lantao Liu --- Makefile | 2 +- hack/install-deps.sh | 154 --------------------- hack/install/install-cni.sh | 66 +++++++++ hack/install/install-containerd.sh | 48 +++++++ hack/install/install-crictl.sh | 40 ++++++ hack/install/install-deps.sh | 50 +++++++ hack/install/install-runc.sh | 38 +++++ hack/install/utils.sh | 54 ++++++++ hack/release.sh | 2 +- hack/sync-vendor.sh | 1 + hack/test-cri.sh | 1 - hack/test-e2e-node.sh | 6 + hack/utils.sh | 17 +-- test/build-containerd.sh | 46 ++++++ test/build-utils.sh | 42 ++++++ test/build.sh | 34 +---- test/configure.sh | 2 +- test/e2e_node/image-config-containerd.yaml | 9 ++ 18 files changed, 412 insertions(+), 200 deletions(-) delete mode 100755 hack/install-deps.sh create mode 100755 hack/install/install-cni.sh create mode 100755 hack/install/install-containerd.sh create mode 100755 hack/install/install-crictl.sh create mode 100755 hack/install/install-deps.sh create mode 100755 hack/install/install-runc.sh create mode 100755 hack/install/utils.sh create mode 100755 test/build-containerd.sh create mode 100755 test/build-utils.sh create mode 100644 test/e2e_node/image-config-containerd.yaml diff --git a/Makefile b/Makefile index 6bb59ea96..aa011ad88 100644 --- a/Makefile +++ b/Makefile @@ -161,7 +161,7 @@ proto: .PHONY: install.deps install.deps: - @./hack/install-deps.sh + @./hack/install/install-deps.sh .PHONY: .gitvalidation # When this is running in travis, it will only check the travis commit range. diff --git a/hack/install-deps.sh b/hack/install-deps.sh deleted file mode 100755 index 415bcfe7a..000000000 --- a/hack/install-deps.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Dependencies: -# runc: -# - libseccomp-dev(Ubuntu,Debian)/libseccomp-devel(Fedora, CentOS, RHEL). Note that -# libseccomp in ubuntu <=trusty and debian <=jessie is not new enough, backport -# is required. -# - libapparmor-dev(Ubuntu,Debian)/libapparmor-devel(Fedora, CentOS, RHEL) -# containerd: -# - btrfs-tools(Ubuntu,Debian)/btrfs-progs-devel(Fedora, CentOS, RHEL) - -set -o errexit -set -o nounset -set -o pipefail - -source $(dirname "${BASH_SOURCE[0]}")/utils.sh - -ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. - -# DESTDIR is the dest path to install dependencies. -DESTDIR=${DESTDIR:-"/"} -# Convert to absolute path if it's relative. -if [[ ${DESTDIR} != /* ]]; then - DESTDIR=${ROOT}/${DESTDIR} -fi - -# NOSUDO indicates not to use sudo during installation. -NOSUDO=${NOSUDO:-false} -sudo="sudo" -if ${NOSUDO}; then - sudo="" -fi - -# INSTALL_CNI indicates whether to install CNI. CNI installation -# makes sense for local testing, but doesn't make sense for cluster -# setup, because CNI daemonset is usually used to deploy CNI binaries -# and configurations in cluster. -INSTALL_CNI=${INSTALL_CNI:-true} - -CONTAINERD_DIR=${DESTDIR}/usr/local -RUNC_DIR=${DESTDIR} -CNI_DIR=${DESTDIR}/opt/cni -CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d -CRICTL_DIR=${DESTDIR}/usr/local/bin -CRICTL_CONFIG_DIR=${DESTDIR}/etc - -RUNC_PKG=github.com/opencontainers/runc -CNI_PKG=github.com/containernetworking/plugins -CONTAINERD_PKG=github.com/containerd/containerd -CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools -CRI_CONTAINERD_PKG=github.com/containerd/cri - -# Create a temporary GOPATH for make install.deps. -TMPGOPATH=$(mktemp -d /tmp/cri-containerd-install-deps.XXXX) -GOPATH=${TMPGOPATH} - -# checkout_repo checks out specified repository -# and switch to specified version. -# Varset: -# 1) Pkg name; -# 2) Version; -# 3) Repo name; -checkout_repo() { - local -r pkg=$1 - local -r version=$2 - local -r repo=$3 - path="${GOPATH}/src/${pkg}" - if [ ! -d ${path} ]; then - mkdir -p ${path} - git clone https://${repo} ${path} - fi - cd ${path} - git fetch --all - git checkout ${version} -} - -# Install runc -checkout_repo ${RUNC_PKG} ${RUNC_VERSION} ${RUNC_REPO} -cd ${GOPATH}/src/${RUNC_PKG} -BUILDTAGS=${BUILDTAGS:-seccomp apparmor} -make static BUILDTAGS="$BUILDTAGS" -${sudo} make install -e DESTDIR=${RUNC_DIR} - -# Install cni -if ${INSTALL_CNI}; then - checkout_repo ${CNI_PKG} ${CNI_VERSION} ${CNI_REPO} - cd ${GOPATH}/src/${CNI_PKG} - FASTBUILD=true ./build.sh - ${sudo} mkdir -p ${CNI_DIR} - ${sudo} cp -r ./bin ${CNI_DIR} - ${sudo} mkdir -p ${CNI_CONFIG_DIR} - ${sudo} bash -c 'cat >'${CNI_CONFIG_DIR}'/10-containerd-net.conflist <'${CRICTL_CONFIG_DIR}'/crictl.yaml <'${CNI_CONFIG_DIR}'/10-containerd-net.conflist <'${CRICTL_CONFIG_DIR}'/crictl.yaml < ${containerd_vendor} # Create a temporary vendor file to update. tmp_vendor=$(mktemp /tmp/vendor.conf.XXXX) diff --git a/hack/test-cri.sh b/hack/test-cri.sh index f34aec0d4..320359598 100755 --- a/hack/test-cri.sh +++ b/hack/test-cri.sh @@ -36,7 +36,6 @@ fi GOPATH=${GOPATH%%:*} CRITEST=${GOPATH}/bin/critest -CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools # Install critest if [ ! -x "$(command -v ${CRITEST})" ]; then diff --git a/hack/test-e2e-node.sh b/hack/test-e2e-node.sh index 0d411fc93..efdae895a 100755 --- a/hack/test-e2e-node.sh +++ b/hack/test-e2e-node.sh @@ -59,6 +59,12 @@ fi GOPATH=${GOPATH%%:*} # Get kubernetes +from-vendor KUBERNETES k8s.io/kubernetes +# k8s.io is actually a redirect, but we do not handle the go-import +# metadata which `go get` and `vndr` etc do. Handle it manually here. +if [ x"$KUBERNETES_REPO" = "xk8s.io" ] ; then + KUBERNETES_REPO="https://github.com/kubernetes/kubernetes" +fi KUBERNETES_PATH="${GOPATH}/src/k8s.io/kubernetes" if [ ! -d "${KUBERNETES_PATH}" ]; then mkdir -p ${KUBERNETES_PATH} diff --git a/hack/utils.sh b/hack/utils.sh index 86d339c08..074b953a5 100755 --- a/hack/utils.sh +++ b/hack/utils.sh @@ -18,8 +18,12 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. # Not from vendor.conf. CRITOOL_VERSION=b184f9aefe60a4441330e615ee20634ee26474fb +CRITOOL_PKG=github.com/kubernetes-incubator/cri-tools CRITOOL_REPO=github.com/kubernetes-incubator/cri-tools +# VENDOR is the path to vendor.conf. +VENDOR=${VENDOR:-"${ROOT}/vendor.conf"} + # upload_logs_to_gcs uploads test logs to gcs. # Var set: # 1. Bucket: gcs bucket to upload logs. @@ -72,7 +76,7 @@ sha256() { from-vendor() { local what=$1 local repo=$2 - local vendor=$(dirname "${BASH_SOURCE[0]}")/../vendor.conf + local vendor=$VENDOR setvars=$(awk -v REPO=$repo -v WHAT=$what -- ' BEGIN { rc=1 } # Assume we did not find what we were looking for. // { @@ -91,14 +95,3 @@ from-vendor() { fi eval $setvars } - -from-vendor RUNC github.com/opencontainers/runc -from-vendor CNI github.com/containernetworking/plugins -from-vendor CONTAINERD github.com/containerd/containerd -from-vendor KUBERNETES k8s.io/kubernetes - -# k8s.io is actually a redirect, but we do not handle the go-import -# metadata which `go get` and `vndr` etc do. Handle it manually here. -if [ x"$KUBERNETES_REPO" = "xk8s.io" ] ; then - KUBERNETES_REPO="https://github.com/kubernetes/kubernetes" -fi diff --git a/test/build-containerd.sh b/test/build-containerd.sh new file mode 100755 index 000000000..1cff039a4 --- /dev/null +++ b/test/build-containerd.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2018 The containerd Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to build and upload latest containerd from +# containerd/containerd in gcr.io/k8s-testimages/kubekins-e2e. + +set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +source $(dirname "${BASH_SOURCE[0]}")/build-utils.sh +cd "${ROOT}" + +if [ -z "${GOPATH}" ]; then + echo "GOPATH is not set" + exit 1 +fi + +CONTAINERD_PATH=${GOPATH}/src/github.com/containerd/containerd +if [ ! -d "${CONTAINERD_PATH}" ]; then + echo "containerd repo does not exist" + exit 1 +fi + +# Make sure output directory is clean. +make clean +# Build and push test tarball. +PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \ + make push TARBALL_PREFIX=containerd-cni \ + INCLUDE_CNI=true \ + CHECKOUT_CONTAINERD=false \ + VENDOR=${CONTAINERD_PATH}/vendor.conf diff --git a/test/build-utils.sh b/test/build-utils.sh new file mode 100755 index 000000000..b50c19837 --- /dev/null +++ b/test/build-utils.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2018 The containerd Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. + +# PROJECT is the gce project to upload tarball. +PROJECT=${PROJECT:-"k8s-cri-containerd"} + +# GOOGLE_APPLICATION_CREDENTIALS is the path of service account file. +if [ -z ${GOOGLE_APPLICATION_CREDENTIALS} ]; then + echo "GOOGLE_APPLICATION_CREDENTIALS is not set" + exit 1 +fi + +# Activate gcloud service account. +gcloud auth activate-service-account --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" --project="${PROJECT}" + +# Install dependent libraries. +sh -c "echo 'deb http://ftp.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/backports.list" +apt-get update +apt-get install -y btrfs-tools +apt-get install -y libseccomp2/jessie-backports +apt-get install -y libseccomp-dev/jessie-backports +apt-get install -y libapparmor-dev + +# PULL_REFS is from prow. +if [ ! -z "${PULL_REFS:-""}" ]; then + DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}') +fi diff --git a/test/build.sh b/test/build.sh index d246d1663..76aaa9e59 100755 --- a/test/build.sh +++ b/test/build.sh @@ -14,45 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script is used to build and upload cri-containerd in gcr.io/k8s-testimages/kubekins-e2e. +# This script is used to build and upload containerd with latest CRI plugin +# from containerd/cri in gcr.io/k8s-testimages/kubekins-e2e. set -o xtrace set -o errexit set -o nounset set -o pipefail -ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. +source $(dirname "${BASH_SOURCE[0]}")/build-utils.sh cd "${ROOT}" -# PROJECT is the gce project to upload tarball. -PROJECT=${PROJECT:-"k8s-cri-containerd"} - -# GOOGLE_APPLICATION_CREDENTIALS is the path of service account file. -if [ -z ${GOOGLE_APPLICATION_CREDENTIALS} ]; then - echo "GOOGLE_APPLICATION_CREDENTIALS is not set" - exit 1 -fi - -# Activate gcloud service account. -gcloud auth activate-service-account --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" --project="${PROJECT}" - -# Install dependent libraries. -sh -c "echo 'deb http://ftp.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/backports.list" -apt-get update -apt-get install -y btrfs-tools -apt-get install -y libseccomp2/jessie-backports -apt-get install -y libseccomp-dev/jessie-backports -apt-get install -y libapparmor-dev - -# PULL_REFS is from prow. -if [ ! -z "${PULL_REFS:-""}" ]; then - DEPLOY_DIR=$(echo "${PULL_REFS}" | sha1sum | awk '{print $1}') -fi - # Make sure output directory is clean. make clean -# Build and push e2e tarball. -DEPLOY_DIR=${DEPLOY_DIR:-""} make push -# Build and push node e2e tarball. +# Build and push test tarball. PUSH_VERSION=true DEPLOY_DIR=${DEPLOY_DIR:-""} \ make push TARBALL_PREFIX=cri-containerd-cni INCLUDE_CNI=true CUSTOM_CONTAINERD=true diff --git a/test/configure.sh b/test/configure.sh index c33d8b71e..f35f4925a 100755 --- a/test/configure.sh +++ b/test/configure.sh @@ -50,7 +50,7 @@ fi # By default use the release tarball with cni built in. PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"} # PKG_PREFIX_METADATA is the metadata key of PKG_PREFIX. -PKG_PREFIX_METADATA="pkg_prefix" +PKG_PREFIX_METADATA="pkg-prefix" pkg_prefix=$(fetch_metadata "${PKG_PREFIX_METADATA}") if [ ! -z "${pkg_prefix}" ]; then PKG_PREFIX=${pkg_prefix} diff --git a/test/e2e_node/image-config-containerd.yaml b/test/e2e_node/image-config-containerd.yaml new file mode 100644 index 000000000..a3960cc5f --- /dev/null +++ b/test/e2e_node/image-config-containerd.yaml @@ -0,0 +1,9 @@ +images: + ubuntu: + image: ubuntu-gke-1604-xenial-v20170420-1 + project: ubuntu-os-gke-cloud + metadata: "user-data