Merge pull request #67971 from Katharine/coverage-instrumentation

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

Add ability to build with runtime coverage instrumentation

**What this PR does / why we need it**:

This PR adds the ability to instrument a subset of kubernetes binaries to report code coverage information. The specific use-case is to help determine coverage of our end-to-end Conformance tests, as well as provide data that can be used to help determine where to focus. This PR focuses on making it possible to build with instrumentation; collecting and using the generated coverage data will be done in later PRs. For more details as to the intent, see the [design doc](https://docs.google.com/document/d/1FKMBFxz7vtA-6ZgUkA47F8m6yR00fwqLcXMVJqsHt0g/edit?usp=sharing) (google doc; requires kubernetes-dev membership).

Specifically, this PR adds a new `KUBE_BUILD_WITH_COVERAGE` make variable, which when set will cause `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, `kube-proxy` and `kubelet` to be built with coverage instrumentation. These coverage-instrumented binaries will flush coverage information to disk every five seconds, defaulting to a temporary directory unless the `KUBE_COVERAGE_FILE` environment variable is set at launch, in which case it will write to that file instead.

The mechanism used to achieve coverage instrumentation is to build the targeted binaries as "unit tests" with coverage enabled, and then rigging the unit tests to just execute the binary's usual entry point. This is implemented only for the bash build system.

/sig testing

```release-note
NONE
```

hack/lib/golang.sh

@@ -228,6 +228,15 @@ readonly KUBE_STATIC_LIBRARIES=(
   kubectl
 )
 
+# Fully-qualified package names that we want to instrument for coverage information.
+readonly KUBE_COVERAGE_INSTRUMENTED_PACKAGES=(
+  k8s.io/kubernetes/cmd/kube-apiserver
+  k8s.io/kubernetes/cmd/kube-controller-manager
+  k8s.io/kubernetes/cmd/kube-scheduler
+  k8s.io/kubernetes/cmd/kube-proxy
+  k8s.io/kubernetes/cmd/kubelet
+)
+
 # KUBE_CGO_OVERRIDES is a space-separated list of binaries which should be built
 # with CGO enabled, assuming CGO is supported on the target platform.
 # This overrides any entry in KUBE_STATIC_LIBRARIES.
@@ -458,6 +467,100 @@ kube::golang::outfile_for_binary() {
   echo "${output_path}/${bin}"
 }
 
+# Argument: the name of a Kubernetes package.
+# Returns 0 if the binary can be built with coverage, 1 otherwise.
+# NB: this ignores whether coverage is globally enabled or not.
+kube::golang::is_instrumented_package() {
+  return $(kube::util::array_contains "$1" "${KUBE_COVERAGE_INSTRUMENTED_PACKAGES[@]}")
+}
+
+# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler)
+# Echos the path to a dummy test used for coverage information.
+kube::golang::path_for_coverage_dummy_test() {
+  local package="$1"
+  local path="${KUBE_GOPATH}/src/${package}"
+  local name=$(basename "${package}")
+  echo "$path/zz_generated_${name}_test.go"
+}
+
+# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler).
+# Creates a dummy unit test on disk in the source directory for the given package.
+# This unit test will invoke the package's standard entry point when run.
+kube::golang::create_coverage_dummy_test() {
+  local package="$1"
+  local name="$(basename "${package}")"
+  cat <<EOF > $(kube::golang::path_for_coverage_dummy_test "${package}")
+package main
+import (
+  "testing"
+  "k8s.io/kubernetes/pkg/util/coverage"
+)
+
+func TestMain(m *testing.M) {
+  // Get coverage running
+  coverage.InitCoverage("${name}")
+
+  // Go!
+  main()
+
+  // Make sure we actually write the profiling information to disk, if we make it here.
+  // On long-running services, or anything that calls os.Exit(), this is insufficient,
+  // so we also flush periodically with a default period of five seconds (configurable by
+  // the KUBE_COVERAGE_FLUSH_INTERVAL environment variable).
+  coverage.FlushCoverage()
+}
+EOF
+}
+
+# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler).
+# Deletes a test generated by kube::golang::create_coverage_dummy_test.
+# It is not an error to call this for a nonexistent test.
+kube::golang::delete_coverage_dummy_test() {
+  local package="$1"
+  rm -f $(kube::golang::path_for_coverage_dummy_test "${package}")
+}
+
+# Arguments: a list of kubernetes packages to build.
+# Expected variables: ${build_args} should be set to an array of Go build arguments.
+# In addition, ${package} and ${platform} should have been set earlier, and if
+# ${build_with_coverage} is set, coverage instrumentation will be enabled.
+#
+# Invokes Go to actually build some packages. If coverage is disabled, simply invokes
+# go install. If coverage is enabled, builds covered binaries using go test, temporarily
+# producing the required unit test files and then cleaning up after itself.
+# Non-covered binaries are then built using go install as usual.
+kube::golang::build_some_binaries() {
+  if [[ -n "${build_with_coverage:-}" ]]; then
+    local -a uncovered=()
+    for package in "$@"; do
+      if kube::golang::is_instrumented_package "${package}"; then
+        V=2 kube::log::info "Building ${package} with coverage..."
+
+        kube::golang::create_coverage_dummy_test "${package}"
+        kube::util::trap_add "kube::golang::delete_coverage_dummy_test \"${package}\"" EXIT
+
+        go test -c -o "$(kube::golang::outfile_for_binary "${package}" "${platform}")" \
+          -covermode count \
+          -coverpkg k8s.io/... \
+          "${build_args[@]}" \
+          -tags coverage \
+          "${package}"
+      else
+        uncovered+=("${package}")
+      fi
+    done
+    if [[ "${#uncovered[@]}" != 0 ]]; then
+      V=2 kube::log::info "Building ${uncovered[@]} without coverage..."
+      go install "${build_args[@]}" "${uncovered[@]}"
+    else
+      V=2 kube::log::info "Nothing to build without coverage."
+     fi
+   else
+    V=2 kube::log::info "Coverage is disabled."
+    go install "${build_args[@]}" "$@"
+   fi
+}
+
 kube::golang::build_binaries_for_platform() {
   local platform=$1
 
@@ -477,18 +580,24 @@ kube::golang::build_binaries_for_platform() {
     fi
   done
 
+  local -a build_args
   if [[ "${#statics[@]}" != 0 ]]; then
-    CGO_ENABLED=0 go install -installsuffix static "${goflags[@]:+${goflags[@]}}" \
-      -gcflags "${gogcflags}" \
-      -ldflags "${goldflags}" \
-      "${statics[@]:+${statics[@]}}"
+    build_args=(
+      -installsuffix static
+      ${goflags:+"${goflags[@]}"}
+      -gcflags "${gogcflags:-}"
+      -ldflags "${goldflags:-}"
+    )
+    CGO_ENABLED=0 kube::golang::build_some_binaries "${statics[@]}"
   fi
 
   if [[ "${#nonstatics[@]}" != 0 ]]; then
-    go install "${goflags[@]:+${goflags[@]}}" \
-      -gcflags "${gogcflags}" \
-      -ldflags "${goldflags}" \
-      "${nonstatics[@]:+${nonstatics[@]}}"
+    build_args=(
+      ${goflags:+"${goflags[@]}"}
+      -gcflags "${gogcflags:-}"
+      -ldflags "${goldflags:-}"
+    )
+    kube::golang::build_some_binaries "${nonstatics[@]}"
   fi
 
   for test in "${tests[@]:+${tests[@]}}"; do
@@ -497,9 +606,9 @@ kube::golang::build_binaries_for_platform() {
 
     mkdir -p "$(dirname ${outfile})"
     go test -c \
-      "${goflags[@]:+${goflags[@]}}" \
-      -gcflags "${gogcflags}" \
-      -ldflags "${goldflags}" \
+      ${goflags:+"${goflags[@]}"} \
+      -gcflags "${gogcflags:-}" \
+      -ldflags "${goldflags:-}" \
       -o "${outfile}" \
       "${testpkg}"
   done
@@ -552,10 +661,11 @@ kube::golang::build_binaries() {
     host_platform=$(kube::golang::host_platform)
 
     # Use eval to preserve embedded quoted strings.
-    local goflags goldflags gogcflags
+    local goflags goldflags gogcflags build_with_coverage
     eval "goflags=(${GOFLAGS:-})"
     goldflags="${GOLDFLAGS:-} $(kube::version::ldflags)"
     gogcflags="${GOGCFLAGS:-}"
+    build_with_coverage="${KUBE_BUILD_WITH_COVERAGE:-}"
 
     local -a targets=()
     local arg