Merge pull request #30546 from thockin/build-vol-whole-output

Automatic merge from submit-queue

Fix subtle build breakage

Repro case:
$ make clean generated_files
$ hack/update-generated-protobuf.sh

This would complain about not finding `fmt`, and it was indicating the wrong
GOROOT.  The problem was that the first step built binaries for generating
code, which *embeds* the value of GOROOT into the binary.  The whole tree was
bind-mounted into the build container and then JUST the dockerized dir was
mounted over it.  The in-container build tried to use the existing binaries,
but GOROOT is wrong.

This change whites-out the whole _output dir.

I first made just an anonymous volume for _output, but docker makes that as
root, which means I can't write to it from our non-root build.  So I just put
it in the data container.  This seems to work.  The biggest change this makes
is that the $GOPATH/bin/ and $GOPATH/pkg/ dirs will persist across dockerized
builds.

NB: this requires a `make clean` to activate.

@lavalamp @jbeda @quinton-hoole @david-mcmahon

build/common.sh

@@ -67,15 +67,15 @@ readonly LOCAL_OUTPUT_IMAGE_STAGING="${LOCAL_OUTPUT_ROOT}/images"
 # This is a symlink to binaries for "this platform" (e.g. build tools).
 readonly THIS_PLATFORM_BIN="${LOCAL_OUTPUT_ROOT}/bin"
 
-readonly OUTPUT_BINPATH="${CUSTOM_OUTPUT_BINPATH:-$LOCAL_OUTPUT_BINPATH}"
-
 readonly REMOTE_OUTPUT_ROOT="/go/src/${KUBE_GO_PACKAGE}/_output"
 readonly REMOTE_OUTPUT_SUBPATH="${REMOTE_OUTPUT_ROOT}/dockerized"
 readonly REMOTE_OUTPUT_BINPATH="${REMOTE_OUTPUT_SUBPATH}/bin"
 readonly REMOTE_OUTPUT_GOPATH="${REMOTE_OUTPUT_SUBPATH}/go"
 
 readonly DOCKER_MOUNT_ARGS_BASE=(
-  --volume "${OUTPUT_BINPATH}:${REMOTE_OUTPUT_BINPATH}"
+  # where the container build will drop output
+  --volume "${LOCAL_OUTPUT_BINPATH}:${REMOTE_OUTPUT_BINPATH}"
+  # timezone
   --volume /etc/localtime:/etc/localtime:ro
 )
 
@@ -568,11 +568,12 @@ function kube::build::ensure_data_container() {
     # container and chowns the GOPATH.
     local -ra docker_cmd=(
       "${DOCKER[@]}" run
-      --volume "${REMOTE_OUTPUT_GOPATH}"
       --name "${KUBE_BUILD_DATA_CONTAINER_NAME}"
       --hostname "${HOSTNAME}"
+      --volume "${REMOTE_OUTPUT_ROOT}"   # white-out the whole output dir
+      --volume "${REMOTE_OUTPUT_GOPATH}" # make a non-root owned mountpoint
       "${KUBE_BUILD_IMAGE}"
-      chown -R $(id -u).$(id -g) "${REMOTE_OUTPUT_GOPATH}"
+      chown -R $(id -u).$(id -g) "${REMOTE_OUTPUT_ROOT}"
     )
     "${docker_cmd[@]}"
   fi