hack/versions: extract most from vendor.conf

This sets `$what_VERSION` and `$what_REPO` for runc, cni, containerd and
kubernetes based on vendor.conf, removing the need to duplicate things in
hack/versions.

With this `update_hack_versions` becomes redundant so remove it and both calls.

Since CONTAINERD_REPO is now unconditionally set we can also simplify the
fetching of vendor.conf in update-vendor.sh a bit, so do so. Further since
`*_REPO` are now unconditionally set we can support alternative clone paths for
all of these repos by adjusting checkout_repo to make the 3rd argument
non-optional and always passing it. Since `CRITOOL_VERSION` is not coming from
`vendor.conf` (since it is not used from Go code) we manually set
`CRITOOL_REPO` for consistency.

The final wrinkle is that `k8s.io/kubernetes` is has a Go specific redirect in
the form of HTML meta headers returned from https://k8s.io/kubernetes/?go-get=1
which point to the real repo to clone. Parsing that in shell is tricky so just
hardcode that.

Fixes #540.

Signed-off-by: Ian Campbell <ijc@docker.com>

hack/install-deps.sh

@@ -27,6 +27,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+source $(dirname "${BASH_SOURCE[0]}")/utils.sh
+
 ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
 . ${ROOT}/hack/versions
 
@@ -76,15 +78,12 @@ GOPATH=${TMPGOPATH}
 # and switch to specified  version.
 # Varset:
 # 1) Pkg name;
-# 2) Version.
+# 2) Version;
-# 3) Repo name (optional);
+# 3) Repo name;
 checkout_repo() {
   local -r pkg=$1
   local -r version=$2
-  local repo=${3:-""}
+  local -r repo=$3
-  if [ -z "${repo}" ]; then
-    repo=${pkg}
-  fi
   path="${GOPATH}/src/${pkg}"
   if [ ! -d ${path} ]; then
     mkdir -p ${path}
@@ -96,7 +95,7 @@ checkout_repo() {
 }
 
 # Install runc
-checkout_repo ${RUNC_PKG} ${RUNC_VERSION}
+checkout_repo ${RUNC_PKG} ${RUNC_VERSION} ${RUNC_REPO}
 cd ${GOPATH}/src/${RUNC_PKG}
 BUILDTAGS=${BUILDTAGS:-seccomp apparmor}
 make static BUILDTAGS="$BUILDTAGS"
@@ -104,7 +103,7 @@ ${sudo} make install -e DESTDIR=${RUNC_DIR}
 
 # Install cni
 if ${INSTALL_CNI}; then
-  checkout_repo ${CNI_PKG} ${CNI_VERSION}
+  checkout_repo ${CNI_PKG} ${CNI_VERSION} ${CNI_REPO}
   cd ${GOPATH}/src/${CNI_PKG}
   FASTBUILD=true ./build.sh
   ${sudo} mkdir -p ${CNI_DIR}
@@ -175,7 +174,7 @@ make BUILDTAGS="${BUILDTAGS}"
 ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}"
 
 #Install crictl
-checkout_repo ${CRITOOL_PKG} ${CRITOOL_VERSION}
+checkout_repo ${CRITOOL_PKG} ${CRITOOL_VERSION} ${CRITOOL_REPO}
 cd ${GOPATH}/src/${CRITOOL_PKG}
 make crictl
 ${sudo} make install-crictl -e BINDIR=${CRICTL_DIR} GOPATH=${GOPATH}

hack/test-e2e-node.sh

@@ -59,12 +59,11 @@ fi
 GOPATH=${GOPATH%%:*}
 
 # Get kubernetes
-KUBERNETES_REPO="https://github.com/kubernetes/kubernetes"
 KUBERNETES_PATH="${GOPATH}/src/k8s.io/kubernetes"
 if [ ! -d "${KUBERNETES_PATH}" ]; then
   mkdir -p ${KUBERNETES_PATH}
   cd ${KUBERNETES_PATH}
-  git clone ${KUBERNETES_REPO} .
+  git clone https://${KUBERNETES_REPO} .
 fi
 cd ${KUBERNETES_PATH}
 git fetch --all

hack/update-vendor.sh

@@ -21,47 +21,11 @@ set -o pipefail
 source $(dirname "${BASH_SOURCE[0]}")/utils.sh
 cd ${ROOT}
 
-update_hack_versions() {
-  need_update=false
-  declare -A map=()
-  map["RUNC_VERSION"]="github.com/opencontainers/runc"
-  map["CNI_VERSION"]="github.com/containernetworking/cni"
-  map["CONTAINERD_VERSION"]="github.com/containerd/containerd"
-  map["KUBERNETES_VERSION"]="k8s.io/kubernetes"
-  for key in ${!map[@]}
-  do
-    vendor_commitid=$(grep ${map[${key}]} vendor.conf | awk '{print $2}')
-    version_commitid=$(grep ${key} hack/versions | awk -F "=" '{print $2}')
-    if [ ${vendor_commitid} != ${version_commitid} ]; then
-      if [ $# -gt 0 ] && [ ${1} = "-only-verify" ]; then
-	need_update=true
-	echo "Need to update the value of ${key} from ${version_commitid} to ${vendor_commitid}."
-      else
-	echo "Updating the value of ${key} from ${version_commitid} to ${vendor_commitid}."
-	sed -i "s/\b${version_commitid}$/${vendor_commitid}/g" hack/versions
-      fi
-    fi
-  done
-
-  if [ ${need_update} = true ]; then
-    echo "Please update \"hack/versions\" by executing \"hack/update-vendor.sh\"!"
-    exit 1
-  fi
-}
-
-echo "Compare vendor with hack/versions..."
-update_hack_versions
-
 # hack/versions should be correct now.
 echo "Compare vendor with containerd vendors..."
 source hack/versions
-if [ -z "${CONTAINERD_REPO}" ]; then
-  CONTAINERD_REPO=containerd/containerd
-else
-  CONTAINERD_REPO=${CONTAINERD_REPO#*/}
-fi
 containerd_vendor=$(mktemp /tmp/containerd-vendor.conf.XXXX)
-curl -s https://raw.githubusercontent.com/${CONTAINERD_REPO}/${CONTAINERD_VERSION}/vendor.conf > ${containerd_vendor}
+curl -s https://raw.githubusercontent.com/${CONTAINERD_REPO#*/}/${CONTAINERD_VERSION}/vendor.conf > ${containerd_vendor}
 # Create a temporary vendor file to update.
 tmp_vendor=$(mktemp /tmp/vendor.conf.XXXX)
 while read vendor; do
@@ -95,9 +59,6 @@ if ! diff vendor.conf ${tmp_vendor} > /dev/null; then
 fi
 rm ${containerd_vendor}
 
-echo "Compare new vendor with hack/versions..."
-update_hack_versions
-
 echo "Sort vendor.conf..."
 sort vendor.conf -o vendor.conf
 

hack/utils.sh

@@ -61,3 +61,29 @@ sha256() {
     shasum -a256 "$1" | awk '{ print $1 }'
   fi
 }
+
+# Takes a prefix ($what) and a $repo and sets `$what_VERSION` and
+# `$what_REPO` from vendor.conf, where `$what_REPO` defaults to $repo
+# but is overridden by the 3rd field of vendor.conf.
+from-vendor() {
+    local what=$1
+    local repo=$2
+    local vendor=$(dirname "${BASH_SOURCE[0]}")/../vendor.conf
+    setvars=$(awk -v REPO=$repo -v WHAT=$what -- '
+                  BEGIN { rc=1 }                        # Assume we did not find what we were looking for.
+                  // {
+                     if ($1 == REPO) {
+                        if ($3 != "") { REPO = $3 };    # Override repo.
+                        printf("%s_VERSION=%s; %s_REPO=%s\n", WHAT, $2, WHAT, REPO);
+                        rc=0;                           # Note success for use in END block.
+                        exit                            # No point looking further.
+                     }
+                 }
+                 END { exit rc }                        # Exit with the desired code.
+                 ' $vendor)
+    if [ $? -ne 0 ] ; then
+        echo "failed to get version of $repo from $vendor" >&2
+        exit 1
+    fi
+    eval $setvars
+}

hack/versions

@@ -1,6 +1,14 @@
-RUNC_VERSION=9f9c96235cc97674e935002fc3d78361b696a69e
+from-vendor RUNC       github.com/opencontainers/runc
-CNI_VERSION=v0.6.0
+from-vendor CNI        github.com/containernetworking/plugins
-CONTAINERD_VERSION=f12ba2407e328c98f8be5eacbb9c510b073dd4c0
+from-vendor CONTAINERD github.com/containerd/containerd
-CONTAINERD_REPO=
+from-vendor KUBERNETES k8s.io/kubernetes
+
+# k8s.io is actually a redirect, but we do not handle the go-import
+# metadata which `go get` and `vndr` etc do. Handle it manually here.
+if [ x"$KUBERNETES_REPO" = "xk8s.io" ] ; then
+    KUBERNETES_REPO="https://github.com/kubernetes/kubernetes"
+fi
+
+# Not from vendor.conf.
 CRITOOL_VERSION=240a840375cdabb5860c75c99e8b0d0a776006b4
-KUBERNETES_VERSION=0caa20c65f147e15f5545862510eb7e81c42b0a3
+CRITOOL_REPO=github.com/kubernetes-incubator/cri-tools