From f6dd8c9e52d1ce697642f09b306a8b1f238be2cb Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 17 Jan 2018 16:30:50 +0000 Subject: [PATCH] hack/versions: extract most from vendor.conf This sets `$what_VERSION` and `$what_REPO` for runc, cni, containerd and kubernetes based on vendor.conf, removing the need to duplicate things in hack/versions. With this `update_hack_versions` becomes redundant so remove it and both calls. Since CONTAINERD_REPO is now unconditionally set we can also simplify the fetching of vendor.conf in update-vendor.sh a bit, so do so. Further since `*_REPO` are now unconditionally set we can support alternative clone paths for all of these repos by adjusting checkout_repo to make the 3rd argument non-optional and always passing it. Since `CRITOOL_VERSION` is not coming from `vendor.conf` (since it is not used from Go code) we manually set `CRITOOL_REPO` for consistency. The final wrinkle is that `k8s.io/kubernetes` is has a Go specific redirect in the form of HTML meta headers returned from https://k8s.io/kubernetes/?go-get=1 which point to the real repo to clone. Parsing that in shell is tricky so just hardcode that. Fixes #540. Signed-off-by: Ian Campbell --- hack/install-deps.sh | 17 ++++++++--------- hack/test-e2e-node.sh | 3 +-- hack/update-vendor.sh | 41 +---------------------------------------- hack/utils.sh | 26 ++++++++++++++++++++++++++ hack/versions | 18 +++++++++++++----- 5 files changed, 49 insertions(+), 56 deletions(-) diff --git a/hack/install-deps.sh b/hack/install-deps.sh index c2cbb626e..e5e0f1cdc 100755 --- a/hack/install-deps.sh +++ b/hack/install-deps.sh @@ -27,6 +27,8 @@ set -o errexit set -o nounset set -o pipefail +source $(dirname "${BASH_SOURCE[0]}")/utils.sh + ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. . ${ROOT}/hack/versions @@ -76,15 +78,12 @@ GOPATH=${TMPGOPATH} # and switch to specified version. # Varset: # 1) Pkg name; -# 2) Version. -# 3) Repo name (optional); +# 2) Version; +# 3) Repo name; checkout_repo() { local -r pkg=$1 local -r version=$2 - local repo=${3:-""} - if [ -z "${repo}" ]; then - repo=${pkg} - fi + local -r repo=$3 path="${GOPATH}/src/${pkg}" if [ ! -d ${path} ]; then mkdir -p ${path} @@ -96,7 +95,7 @@ checkout_repo() { } # Install runc -checkout_repo ${RUNC_PKG} ${RUNC_VERSION} +checkout_repo ${RUNC_PKG} ${RUNC_VERSION} ${RUNC_REPO} cd ${GOPATH}/src/${RUNC_PKG} BUILDTAGS=${BUILDTAGS:-seccomp apparmor} make static BUILDTAGS="$BUILDTAGS" @@ -104,7 +103,7 @@ ${sudo} make install -e DESTDIR=${RUNC_DIR} # Install cni if ${INSTALL_CNI}; then - checkout_repo ${CNI_PKG} ${CNI_VERSION} + checkout_repo ${CNI_PKG} ${CNI_VERSION} ${CNI_REPO} cd ${GOPATH}/src/${CNI_PKG} FASTBUILD=true ./build.sh ${sudo} mkdir -p ${CNI_DIR} @@ -175,7 +174,7 @@ make BUILDTAGS="${BUILDTAGS}" ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}" #Install crictl -checkout_repo ${CRITOOL_PKG} ${CRITOOL_VERSION} +checkout_repo ${CRITOOL_PKG} ${CRITOOL_VERSION} ${CRITOOL_REPO} cd ${GOPATH}/src/${CRITOOL_PKG} make crictl ${sudo} make install-crictl -e BINDIR=${CRICTL_DIR} GOPATH=${GOPATH} diff --git a/hack/test-e2e-node.sh b/hack/test-e2e-node.sh index 67f635f36..4c0810721 100755 --- a/hack/test-e2e-node.sh +++ b/hack/test-e2e-node.sh @@ -59,12 +59,11 @@ fi GOPATH=${GOPATH%%:*} # Get kubernetes -KUBERNETES_REPO="https://github.com/kubernetes/kubernetes" KUBERNETES_PATH="${GOPATH}/src/k8s.io/kubernetes" if [ ! -d "${KUBERNETES_PATH}" ]; then mkdir -p ${KUBERNETES_PATH} cd ${KUBERNETES_PATH} - git clone ${KUBERNETES_REPO} . + git clone https://${KUBERNETES_REPO} . fi cd ${KUBERNETES_PATH} git fetch --all diff --git a/hack/update-vendor.sh b/hack/update-vendor.sh index deeddd4d8..5d9298247 100755 --- a/hack/update-vendor.sh +++ b/hack/update-vendor.sh @@ -21,47 +21,11 @@ set -o pipefail source $(dirname "${BASH_SOURCE[0]}")/utils.sh cd ${ROOT} -update_hack_versions() { - need_update=false - declare -A map=() - map["RUNC_VERSION"]="github.com/opencontainers/runc" - map["CNI_VERSION"]="github.com/containernetworking/cni" - map["CONTAINERD_VERSION"]="github.com/containerd/containerd" - map["KUBERNETES_VERSION"]="k8s.io/kubernetes" - for key in ${!map[@]} - do - vendor_commitid=$(grep ${map[${key}]} vendor.conf | awk '{print $2}') - version_commitid=$(grep ${key} hack/versions | awk -F "=" '{print $2}') - if [ ${vendor_commitid} != ${version_commitid} ]; then - if [ $# -gt 0 ] && [ ${1} = "-only-verify" ]; then - need_update=true - echo "Need to update the value of ${key} from ${version_commitid} to ${vendor_commitid}." - else - echo "Updating the value of ${key} from ${version_commitid} to ${vendor_commitid}." - sed -i "s/\b${version_commitid}$/${vendor_commitid}/g" hack/versions - fi - fi - done - - if [ ${need_update} = true ]; then - echo "Please update \"hack/versions\" by executing \"hack/update-vendor.sh\"!" - exit 1 - fi -} - -echo "Compare vendor with hack/versions..." -update_hack_versions - # hack/versions should be correct now. echo "Compare vendor with containerd vendors..." source hack/versions -if [ -z "${CONTAINERD_REPO}" ]; then - CONTAINERD_REPO=containerd/containerd -else - CONTAINERD_REPO=${CONTAINERD_REPO#*/} -fi containerd_vendor=$(mktemp /tmp/containerd-vendor.conf.XXXX) -curl -s https://raw.githubusercontent.com/${CONTAINERD_REPO}/${CONTAINERD_VERSION}/vendor.conf > ${containerd_vendor} +curl -s https://raw.githubusercontent.com/${CONTAINERD_REPO#*/}/${CONTAINERD_VERSION}/vendor.conf > ${containerd_vendor} # Create a temporary vendor file to update. tmp_vendor=$(mktemp /tmp/vendor.conf.XXXX) while read vendor; do @@ -95,9 +59,6 @@ if ! diff vendor.conf ${tmp_vendor} > /dev/null; then fi rm ${containerd_vendor} -echo "Compare new vendor with hack/versions..." -update_hack_versions - echo "Sort vendor.conf..." sort vendor.conf -o vendor.conf diff --git a/hack/utils.sh b/hack/utils.sh index eed2285f7..1e765c1a4 100644 --- a/hack/utils.sh +++ b/hack/utils.sh @@ -61,3 +61,29 @@ sha256() { shasum -a256 "$1" | awk '{ print $1 }' fi } + +# Takes a prefix ($what) and a $repo and sets `$what_VERSION` and +# `$what_REPO` from vendor.conf, where `$what_REPO` defaults to $repo +# but is overridden by the 3rd field of vendor.conf. +from-vendor() { + local what=$1 + local repo=$2 + local vendor=$(dirname "${BASH_SOURCE[0]}")/../vendor.conf + setvars=$(awk -v REPO=$repo -v WHAT=$what -- ' + BEGIN { rc=1 } # Assume we did not find what we were looking for. + // { + if ($1 == REPO) { + if ($3 != "") { REPO = $3 }; # Override repo. + printf("%s_VERSION=%s; %s_REPO=%s\n", WHAT, $2, WHAT, REPO); + rc=0; # Note success for use in END block. + exit # No point looking further. + } + } + END { exit rc } # Exit with the desired code. + ' $vendor) + if [ $? -ne 0 ] ; then + echo "failed to get version of $repo from $vendor" >&2 + exit 1 + fi + eval $setvars +} diff --git a/hack/versions b/hack/versions index ad0d73371..99cc8945d 100644 --- a/hack/versions +++ b/hack/versions @@ -1,6 +1,14 @@ -RUNC_VERSION=9f9c96235cc97674e935002fc3d78361b696a69e -CNI_VERSION=v0.6.0 -CONTAINERD_VERSION=f12ba2407e328c98f8be5eacbb9c510b073dd4c0 -CONTAINERD_REPO= +from-vendor RUNC github.com/opencontainers/runc +from-vendor CNI github.com/containernetworking/plugins +from-vendor CONTAINERD github.com/containerd/containerd +from-vendor KUBERNETES k8s.io/kubernetes + +# k8s.io is actually a redirect, but we do not handle the go-import +# metadata which `go get` and `vndr` etc do. Handle it manually here. +if [ x"$KUBERNETES_REPO" = "xk8s.io" ] ; then + KUBERNETES_REPO="https://github.com/kubernetes/kubernetes" +fi + +# Not from vendor.conf. CRITOOL_VERSION=240a840375cdabb5860c75c99e8b0d0a776006b4 -KUBERNETES_VERSION=0caa20c65f147e15f5545862510eb7e81c42b0a3 +CRITOOL_REPO=github.com/kubernetes-incubator/cri-tools