Simpler openapi gen - subprojects do themselves

This involves moving the report files, but it allows me to delete the
indirect variable and indirect array code in update-codgen.  As proud as
I was of figuring that out, I am also ashamed of myself for doing it.

This is my atonement.

api/api-rules/README.md

@@ -1,8 +1,10 @@
 # Existing API Rule Violations
 
-This folder contains the checked-in report file of known API rule violations.
+This folder contains the checked-in report file of known API rule violations
-The file violation\_exceptions.list is used by Make rule during OpenAPI spec generation to make
+for the main Kubernetes repository.  Reports for staging repositories are
-sure that no new API rule violation is introduced into our code base.
+checked in to those repositories.  These files are used during OpenAPI spec
+generation to make sure that no new API rule violation is introduced into our
+code base.
 
 ## API Rule Violation Format
 

hack/update-codegen.sh

@@ -497,47 +497,6 @@ function k8s_tag_files_except() {
     done
 }
 
-# $@: directories to exclude
-# example:
-#    k8s_tag_files_matching foo bat/qux
-function k8s_tag_files_matching() {
-    for f in "${ALL_K8S_TAG_FILES[@]}"; do
-        for x in "$@"; do
-            if [[ "$f" =~ "${x}"/.* ]]; then
-                echo "$f"
-                break
-            fi
-        done
-    done
-}
-
-# $1: the name of a scalar variable to read
-# example:
-#    FOO_VAR="foo value"
-#    BAR_VAR="bar value"
-#    x=FOO
-#    indirect "${x}_VAR" # -> "foo value\n"
-function indirect() {
-    # This is a trick to get bash to indirectly read a variable.
-    # Thanks StackOverflow!
-    local var="$1"
-    echo "${!var}"
-}
-
-# $1: the name of an array variable to read
-#    FOO_ARR=(a b c)
-#    BAR_ARR=(1 2 3)
-#    x=FOO
-#    indirect_array "${x}_ARR" # -> "a\nb\nc\n"
-function indirect_array() {
-    # This is a trick to get bash to indirectly read an array.
-    # Thanks StackOverflow!
-    local arrayname="$1"
-    # shellcheck disable=SC1087 # intentional
-    local tmp="$arrayname[@]"
-    printf -- "%s\n" "${!tmp}"
-}
-
 # OpenAPI generation
 #
 # Any package that wants open-api functions generated must include a
@@ -555,109 +514,35 @@ function codegen::openapi() {
     local gen_openapi_bin
     gen_openapi_bin="$(kube::util::find-binary "openapi-gen")"
 
-    # Standard dirs which all targets need.
+    local output_dir="pkg/generated/openapi"
-    local apimachinery_dirs=(
+    local known_violations_file="api/api-rules/violation_exceptions.list"
-        vendor/k8s.io/apimachinery/pkg/apis/meta/v1
-        vendor/k8s.io/apimachinery/pkg/runtime
-        vendor/k8s.io/apimachinery/pkg/version
-    )
 
-    # These should probably be configured by tags in code-files somewhere.
+    local report_file="${OUT_DIR}/api_violations.report"
-    local targets=(
+    # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
-        KUBE
+    # updated API violations to the known API violation exceptions list.
-        AGGREGATOR
+    if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
-        APIEXTENSIONS
+        report_file="${known_violations_file}"
-        CODEGEN
+    fi
-        SAMPLEAPISERVER
-    )
 
-    # shellcheck disable=SC2034 # used indirectly
+    if [[ "${DBG_CODEGEN}" == 1 ]]; then
-    local KUBE_output_dir="pkg/generated/openapi"
+        kube::log::status "DBG: finding all +k8s:openapi-gen tags"
-    # shellcheck disable=SC2034 # used indirectly
+    fi
-    local KUBE_known_violations_file="api/api-rules/violation_exceptions.list"
+
-    # shellcheck disable=SC2034 # used indirectly
+    local tag_files=()
-    local KUBE_tag_files=()
+    kube::util::read-array tag_files < <(
-    kube::util::read-array KUBE_tag_files < <(
         k8s_tag_files_except \
             vendor/k8s.io/code-generator \
             vendor/k8s.io/sample-apiserver
         )
 
-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_output_dir="staging/src/k8s.io/kube-aggregator/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_known_violations_file="api/api-rules/aggregator_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local AGGREGATOR_tag_files=()
-    kube::util::read-array AGGREGATOR_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/kube-aggregator \
-            "${apimachinery_dirs[@]}"
-        )
-
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_output_dir="staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_known_violations_file="api/api-rules/apiextensions_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local APIEXTENSIONS_tag_files=()
-    kube::util::read-array APIEXTENSIONS_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/apiextensions-apiserver \
-            vendor/k8s.io/api/autoscaling/v1 \
-            "${apimachinery_dirs[@]}"
-        )
-
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_output_dir="staging/src/k8s.io/code-generator/examples/apiserver/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_known_violations_file="api/api-rules/codegen_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local CODEGEN_tag_files=()
-    kube::util::read-array CODEGEN_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/code-generator \
-            "${apimachinery_dirs[@]}"
-        )
-
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_output_dir="staging/src/k8s.io/sample-apiserver/pkg/generated/openapi"
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_known_violations_file="api/api-rules/sample_apiserver_violation_exceptions.list"
-    # shellcheck disable=SC2034 # used indirectly
-    local SAMPLEAPISERVER_tag_files=()
-    kube::util::read-array SAMPLEAPISERVER_tag_files < <(
-        k8s_tag_files_matching \
-            vendor/k8s.io/sample-apiserver \
-            "${apimachinery_dirs[@]}"
-        )
-
-    git_find -z ':(glob)**'/"${output_base}.go" | xargs -0 rm -f
-
-    for prefix in "${targets[@]}"; do
-        local report_file="${OUT_DIR}/${prefix}_violations.report"
-        # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
-        # updated API violations to the known API violation exceptions list.
-        if [[ "${UPDATE_API_KNOWN_VIOLATIONS}" == true ]]; then
-            report_file=$(indirect "${prefix}_known_violations_file")
-        fi
-
-        # 2 lines because shellcheck
-        local output_dir
-        output_dir=$(indirect "${prefix}_output_dir")
-
-        if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: finding all +k8s:openapi-gen tags for ${prefix}"
-        fi
-
     local tag_dirs=()
     kube::util::read-array tag_dirs < <(
-            grep -l --null '+k8s:openapi-gen=' $(indirect_array "${prefix}_tag_files") \
+        grep -l --null '+k8s:openapi-gen=' "${tag_files[@]}" \
             | xargs -0 -n1 dirname \
             | sort -u)
 
     if [[ "${DBG_CODEGEN}" == 1 ]]; then
-            kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs for ${prefix}"
+        kube::log::status "DBG: found ${#tag_dirs[@]} +k8s:openapi-gen tagged dirs"
     fi
 
     local tag_pkgs=()
@@ -665,7 +550,7 @@ function codegen::openapi() {
         tag_pkgs+=("${PRJ_SRC_PATH}/$dir")
     done
 
-        kube::log::status "Generating openapi code for ${prefix}"
+    kube::log::status "Generating openapi code"
     if [[ "${DBG_CODEGEN}" == 1 ]]; then
         kube::log::status "DBG: running ${gen_openapi_bin} for:"
         for dir in "${tag_dirs[@]}"; do
@@ -673,6 +558,8 @@ function codegen::openapi() {
         done
     fi
 
+    git_find -z ':(glob)pkg/generated/**'/"${output_base}.go" | xargs -0 rm -f
+
     ./hack/run-in-gopath.sh "${gen_openapi_bin}" \
         --v "${KUBE_VERBOSE}" \
         --logtostderr \
@@ -684,19 +571,16 @@ function codegen::openapi() {
         "$@"
 
     touch "${report_file}"
-        # 2 lines because shellcheck
+    local known_filename="${known_violations_file}"
-        local known_filename
-        known_filename=$(indirect "${prefix}_known_violations_file")
     if ! diff -u "${known_filename}" "${report_file}"; then
         echo -e "ERROR:"
-            echo -e "\t'${prefix}' API rule check failed - reported violations differ from known violations"
+        echo -e "\tAPI rule check failed - reported violations differ from known violations"
         echo -e "\tPlease read api/api-rules/README.md to resolve the failure in ${known_filename}"
     fi
 
     if [[ "${DBG_CODEGEN}" == 1 ]]; then
         kube::log::status "Generated openapi code"
     fi
-    done # for each prefix
 }
 
 function codegen::applyconfigs() {

staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh

@@ -28,11 +28,17 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
     --input-pkg-root k8s.io/apiextensions-apiserver/pkg \
     --extra-pkgs k8s.io/api/autoscaling/v1 `# needed for Scale type` \
     --output-pkg-root k8s.io/apiextensions-apiserver/pkg/generated \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
 kube::codegen::gen_client \

staging/src/k8s.io/code-generator/examples/hack/update-codegen.sh

@@ -34,10 +34,16 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
     --input-pkg-root k8s.io/code-generator/examples/apiserver/apis \
     --output-pkg-root k8s.io/code-generator/examples/apiserver \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
 kube::codegen::gen_client \

staging/src/k8s.io/code-generator/kube_codegen.sh

@@ -237,6 +237,9 @@ function kube::codegen::gen_helpers() {
 #     An optional path at which to write an API violations report.  "-" means
 #     stdout.
 #
+#   --update-report
+#     If specified, update the report file in place, rather than diffing it.
+#
 #   --boilerplate <string = path_to_kube_codegen_boilerplate>
 #     An optional override for the header file to insert into generated files.
 #
@@ -247,6 +250,7 @@ function kube::codegen::gen_openapi() {
     local openapi_subdir="openapi"
     local extra_pkgs=()
     local report="/dev/null"
+    local update_report=""
     local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt"
     local v="${KUBE_VERBOSE:-0}"
 
@@ -276,6 +280,10 @@ function kube::codegen::gen_openapi() {
                 report="$2"
                 shift 2
                 ;;
+            "--update-report")
+                update_report="true"
+                shift
+                ;;
             "--boilerplate")
                 boilerplate="$2"
                 shift 2
@@ -300,6 +308,12 @@ function kube::codegen::gen_openapi() {
         return 1
     fi
 
+    local new_report
+    new_report="$(mktemp -t "$(basename "$0").api_violations.XXXXXX")"
+    if [ -n "${update_report}" ]; then
+        new_report="${report}"
+    fi
+
     (
         # To support running this from anywhere, first cd into this directory,
         # and then install with forced module mode on and fully qualified name.
@@ -348,12 +362,20 @@ function kube::codegen::gen_openapi() {
             --go-header-file "${boilerplate}" \
             --output-base "${out_base}" \
             --output-package "${out_pkg_root}/${openapi_subdir}" \
-            --report-filename "${report}" \
+            --report-filename "${new_report}" \
             --input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1" \
             --input-dirs "k8s.io/apimachinery/pkg/runtime" \
             --input-dirs "k8s.io/apimachinery/pkg/version" \
             "${inputs[@]}"
     fi
+
+    touch "${report}" # in case it doesn't exist yet
+    if ! diff -u "${report}" "${new_report}"; then
+        echo -e "ERROR:"
+        echo -e "\tAPI rule check failed for ${report}: new reported violations"
+        echo -e "\tPlease read api/api-rules/README.md"
+        return 1
+    fi
 }
 
 # Generate client code

staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh

@@ -28,10 +28,16 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
     --input-pkg-root k8s.io/kube-aggregator/pkg/apis \
     --output-pkg-root k8s.io/kube-aggregator/pkg/generated \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
 kube::codegen::gen_client \

staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh

@@ -33,10 +33,16 @@ kube::codegen::gen_helpers \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
+if [[ "${UPDATE_API_KNOWN_VIOLATIONS:-}" == "true" ]]; then
+    update_report="--update-report"
+fi
+
 kube::codegen::gen_openapi \
     --input-pkg-root k8s.io/sample-apiserver/pkg/apis \
     --output-pkg-root k8s.io/sample-apiserver/pkg/generated \
     --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
+    --report-filename "${SCRIPT_ROOT}/api_violation_exceptions.list" \
+    ${update_report:+"${update_report}"} \
     --boilerplate "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 
 kube::codegen::gen_client \