Merge pull request #84058 from claudiubelu/test-image-publishing

test images: Adds E2E test image automated build

test/images/cloudbuild.yaml

@@ -0,0 +1,29 @@
+# See https://cloud.google.com/cloud-build/docs/build-config
+
+# this must be specified in seconds. If omitted, defaults to 600s (10 mins)
+# the timeout was increased because some images might take longer to build (e.g.: sample-apiserver, pets/redis-installer)
+timeout: 3000s
+# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
+# or any new substitutions added in the future.
+options:
+  substitution_option: ALLOW_LOOSE
+steps:
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4'
+    entrypoint: make
+    env:
+    - DOCKER_CLI_EXPERIMENTAL=enabled
+    - TAG=$_GIT_TAG
+    - BASE_REF=$_PULL_BASE_REF
+    - WHAT=$_WHAT
+    - REGISTRY=gcr.io/k8s-staging-e2e-test-images
+    args:
+    - all-push
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution
+  _GIT_TAG: '12345'
+  # _PULL_BASE_REF will contain the ref that was pushed to to trigger this build -
+  # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'.
+  _PULL_BASE_REF: 'master'
+  # _WHAT will contain the image name to be built and published to the staging registry.
+  _WHAT: 'all-conformance'

test/images/image-util.sh

@@ -19,7 +19,7 @@ set -o nounset
 set -o pipefail
 
 TASK=$1
-IMAGE=$2
+WHAT=$2
 
 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
 source "${KUBE_ROOT}/hack/lib/util.sh"
@@ -29,7 +29,8 @@ declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["pp
 
 # Returns list of all supported architectures from BASEIMAGE file
 listArchs() {
-  cut -d "=" -f 1 "${IMAGE}"/BASEIMAGE
+  image=$1
+  cut -d "=" -f 1 "${image}"/BASEIMAGE
 }
 
 # Returns baseimage need to used in Dockerfile for any given architecture
@@ -43,8 +44,9 @@ getBaseImage() {
 # it will build for all the supported arch list - amd64, arm,
 # arm64, ppc64le, s390x
 build() {
-  if [[ -f ${IMAGE}/BASEIMAGE ]]; then
+  image=$1
-    archs=$(listArchs)
+  if [[ -f ${image}/BASEIMAGE ]]; then
+    archs=$(listArchs "$image")
   else
     archs=${!QEMUARCHS[*]}
   fi
@@ -52,7 +54,7 @@ build() {
   kube::util::ensure-gnu-sed
 
   for arch in ${archs}; do
-    echo "Building image for ${IMAGE} ARCH: ${arch}..."
+    echo "Building image for ${image} ARCH: ${arch}..."
 
     # Create a temporary directory for every architecture and copy the image content
     # and build the image from temporary directory
@@ -60,11 +62,11 @@ build() {
     temp_dir=$(mktemp -d "${KUBE_ROOT}"/_tmp/test-images-build.XXXXXX)
     kube::util::trap_add "rm -rf ${temp_dir}" EXIT
 
-    cp -r "${IMAGE}"/* "${temp_dir}"
+    cp -r "${image}"/* "${temp_dir}"
-    if [[ -f ${IMAGE}/Makefile ]]; then
+    if [[ -f ${image}/Makefile ]]; then
       # make bin will take care of all the prerequisites needed
       # for building the docker image
-      make -C "${IMAGE}" bin ARCH="${arch}" TARGET="${temp_dir}"
+      make -C "${image}" bin ARCH="${arch}" TARGET="${temp_dir}"
     fi
     pushd "${temp_dir}"
     # image tag
@@ -96,7 +98,7 @@ build() {
       fi
     fi
 
-    docker build --pull -t "${REGISTRY}/${IMAGE}-${arch}:${TAG}" .
+    docker build --pull -t "${REGISTRY}/${image}-${arch}:${TAG}" .
 
     popd
   done
@@ -114,28 +116,29 @@ docker_version_check() {
 
 # This function will push the docker images
 push() {
+  image=$1
   docker_version_check
-  TAG=$(<"${IMAGE}"/VERSION)
+  TAG=$(<"${image}"/VERSION)
-  if [[ -f ${IMAGE}/BASEIMAGE ]]; then
+  if [[ -f ${image}/BASEIMAGE ]]; then
-    archs=$(listArchs)
+    archs=$(listArchs "$image")
   else
     archs=${!QEMUARCHS[*]}
   fi
   for arch in ${archs}; do
-    docker push "${REGISTRY}/${IMAGE}-${arch}:${TAG}"
+    docker push "${REGISTRY}/${image}-${arch}:${TAG}"
   done
 
   kube::util::ensure-gnu-sed
 
   # The manifest command is still experimental as of Docker 18.09.2
   export DOCKER_CLI_EXPERIMENTAL="enabled"
-  # Make archs list into image manifest. Eg: 'amd64 ppc64le' to '${REGISTRY}/${IMAGE}-amd64:${TAG} ${REGISTRY}/${IMAGE}-ppc64le:${TAG}'
+  # Make archs list into image manifest. Eg: 'amd64 ppc64le' to '${REGISTRY}/${image}-amd64:${TAG} ${REGISTRY}/${image}-ppc64le:${TAG}'
-  while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$archs" | ${SED} -e "s~[^ ]*~$REGISTRY\/$IMAGE\-&:$TAG~g")
+  while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$archs" | ${SED} -e "s~[^ ]*~$REGISTRY\/$image\-&:$TAG~g")
-  docker manifest create --amend "${REGISTRY}/${IMAGE}:${TAG}" "${manifest[@]}"
+  docker manifest create --amend "${REGISTRY}/${image}:${TAG}" "${manifest[@]}"
   for arch in ${archs}; do
-    docker manifest annotate --arch "${arch}" "${REGISTRY}/${IMAGE}:${TAG}" "${REGISTRY}/${IMAGE}-${arch}:${TAG}"
+    docker manifest annotate --arch "${arch}" "${REGISTRY}/${image}:${TAG}" "${REGISTRY}/${image}-${arch}:${TAG}"
   done
-  docker manifest push --purge "${REGISTRY}/${IMAGE}:${TAG}"
+  docker manifest push --purge "${REGISTRY}/${image}:${TAG}"
 }
 
 # This function is for building the go code
@@ -156,4 +159,18 @@ bin() {
 
 shift
 
-eval "${TASK}" "$@"
+if [[ "${WHAT}" == "all-conformance" ]]; then
+  # NOTE(claudiub): Building *ALL* the images under the kubernetes/test/images folder takes an extremely
+  # long time (especially some images), and some images are rarely used and rarely updated, so there's
+  # no point in rebuilding all of them every time. This will only build the Conformance-related images.
+  # Discussed during Conformance Office Hours Meeting (2019.12.17):
+  # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh
+  # echoserver image not included: https://github.com/kubernetes/kubernetes/issues/84158
+  conformance_images=("agnhost" "dnsutils" "jessie-dnsutils" "kitten" "mounttest" "mounttest-user"\
+    "nautilus" "nonewprivs" "resource-consumer" "resource-consumer-controller" "sample-apiserver" "test-webserver")
+  for image in "${conformance_images[@]}"; do
+    eval "${TASK}" "${image}"
+  done
+else
+  eval "${TASK}" "$@"
+fi