Merge pull request #116280 from thockin/codegen_no_internal_subprojects

Codegen: streamline code for subproject generation

staging/src/k8s.io/apiextensions-apiserver/examples/client-go/hack/update-codegen.sh

@@ -25,7 +25,7 @@ CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-
 # --output-base    because this script should also be able to run inside the vendor dir of
 #                  k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir
 #                  instead of the $GOPATH directly. For normal projects this can be dropped.
-"${CODEGEN_PKG}/generate-groups.sh" all \
+"${CODEGEN_PKG}/generate-groups.sh" "applyconfiguration,client,deepcopy,informer,lister" \
   k8s.io/apiextensions-apiserver/examples/client-go/pkg/client \
   k8s.io/apiextensions-apiserver/examples/client-go/pkg/apis \
   cr:v1 \

staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh

@@ -23,7 +23,7 @@ CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-
 
 CLIENTSET_NAME_VERSIONED=clientset \
 CLIENTSET_PKG_NAME=clientset \
-"${CODEGEN_PKG}/generate-groups.sh" "deepcopy,client,lister,informer" \
+"${CODEGEN_PKG}/generate-groups.sh" "deepcopy,defaulter,client,lister,informer" \
   k8s.io/apiextensions-apiserver/pkg/client \
   k8s.io/apiextensions-apiserver/pkg/apis \
   "apiextensions:v1beta1,v1" \

staging/src/k8s.io/code-generator/generate-groups.sh

@@ -25,7 +25,7 @@ if [ "$#" -lt 4 ] || [ "${1}" == "--help" ]; then
   cat <<EOF
 Usage: $(basename "$0") <generators> <output-package> <apis-package> <groups-versions> ...
 
-  <generators>        the generators comma separated to run (deepcopy,defaulter,applyconfiguration,client,lister,informer) or "all".
+  <generators>        the generators comma separated to run (deepcopy,defaulter,applyconfiguration,client,lister,informer).
   <output-package>    the output package name (e.g. github.com/example/project/pkg/generated).
   <apis-package>      the external types dir (e.g. github.com/example/api or github.com/example/project/pkg/apis).
   <groups-versions>   the groups and their versions in the format "groupA:v1,v2 groupB:v1 groupC:v2", relative
@@ -33,9 +33,12 @@ Usage: $(basename "$0") <generators> <output-package> <apis-package> <groups-ver
   ...                 arbitrary flags passed to all generator binaries.
 
 
-Examples:
+Example:
-  $(basename "$0") all             github.com/example/project/pkg/client github.com/example/project/pkg/apis "foo:v1 bar:v1alpha1,v1beta1"
+  $(basename "$0") \
-  $(basename "$0") deepcopy,client github.com/example/project/pkg/client github.com/example/project/pkg/apis "foo:v1 bar:v1alpha1,v1beta1"
+      deepcopy,client \
+      github.com/example/project/pkg/client \
+      github.com/example/project/pkg/apis \
+      "foo:v1 bar:v1alpha1,v1beta1"
 EOF
   exit 0
 fi
@@ -46,73 +49,14 @@ APIS_PKG="$3"
 GROUPS_WITH_VERSIONS="$4"
 shift 4
 
-(
+if [ "${GENS}" = "all" ] || grep -qw "all" <<<"${GENS}"; then
-  # To support running this script from anywhere, first cd into this directory,
+    ALL="applyconfiguration,client,deepcopy,informer,lister"
-  # and then install with forced module mode on and fully qualified name.
+    echo "WARNING: Specifying \"all\" as a generator is deprecated."
-  cd "$(dirname "${0}")"
+    echo "WARNING: Please list the specific generators needed."
-  GO111MODULE=on go install k8s.io/code-generator/cmd/{applyconfiguration-gen,defaulter-gen,client-gen,lister-gen,informer-gen,deepcopy-gen}
+    echo "WARNING: \"all\" is now an alias for \"${ALL}\"; new code generators WILL NOT be added to this set"
-)
+    echo
-# Go installs the above commands to get installed in $GOBIN if defined, and $GOPATH/bin otherwise:
+    GENS="${ALL}"
-GOBIN="$(go env GOBIN)"
-gobin="${GOBIN:-$(go env GOPATH)/bin}"
-
-function codegen::join() { local IFS="$1"; shift; echo "$*"; }
-
-# enumerate group versions
-FQ_APIS=() # e.g. k8s.io/api/apps/v1
-for GVs in ${GROUPS_WITH_VERSIONS}; do
-  IFS=: read -r G Vs <<<"${GVs}"
-
-  # enumerate versions
-  for V in ${Vs//,/ }; do
-    FQ_APIS+=("${APIS_PKG}/${G}/${V}")
-  done
-done
-
-if [ "${GENS}" = "all" ] || grep -qw "deepcopy" <<<"${GENS}"; then
-  echo "Generating deepcopy funcs"
-  "${gobin}/deepcopy-gen" \
-      --input-dirs "$(codegen::join , "${FQ_APIS[@]}")" \
-      -O zz_generated.deepcopy \
-      "$@"
 fi
 
-if [ "${GENS}" = "all" ] || grep -qw "applyconfiguration" <<<"${GENS}"; then
+INT_APIS_PKG=""
-  echo "Generating apply configuration for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/${APPLYCONFIGURATION_PKG_NAME:-applyconfiguration}"
+exec "$(dirname "${BASH_SOURCE[0]}")/generate-internal-groups.sh" "${GENS}" "${OUTPUT_PKG}" "${INT_APIS_PKG}" "${APIS_PKG}" "${GROUPS_WITH_VERSIONS}" "$@"
-  "${gobin}/applyconfiguration-gen" \
-      --input-dirs "$(codegen::join , "${FQ_APIS[@]}")" \
-      --output-package "${OUTPUT_PKG}/${APPLYCONFIGURATION_PKG_NAME:-applyconfiguration}" \
-      "$@"
-fi
-
-if [ "${GENS}" = "all" ] || grep -qw "client" <<<"${GENS}"; then
-  echo "Generating clientset for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}"
-  if [ "${GENS}" = "all" ] || grep -qw "applyconfiguration" <<<"${GENS}"; then
-    APPLY_CONFIGURATION_PACKAGE="${OUTPUT_PKG}/${APPLYCONFIGURATION_PKG_NAME:-applyconfiguration}"
-  fi
-  "${gobin}/client-gen" \
-      --clientset-name "${CLIENTSET_NAME_VERSIONED:-versioned}" \
-      --input-base "" \
-      --input "$(codegen::join , "${FQ_APIS[@]}")" \
-      --output-package "${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}" \
-      --apply-configuration-package "${APPLY_CONFIGURATION_PACKAGE:-}" \
-      "$@"
-fi
-
-if [ "${GENS}" = "all" ] || grep -qw "lister" <<<"${GENS}"; then
-  echo "Generating listers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/listers"
-  "${gobin}/lister-gen" \
-      --input-dirs "$(codegen::join , "${FQ_APIS[@]}")" \
-      --output-package "${OUTPUT_PKG}/listers" \
-      "$@"
-fi
-
-if [ "${GENS}" = "all" ] || grep -qw "informer" <<<"${GENS}"; then
-  echo "Generating informers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/informers"
-  "${gobin}/informer-gen" \
-      --input-dirs "$(codegen::join , "${FQ_APIS[@]}")" \
-      --versioned-clientset-package "${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}/${CLIENTSET_NAME_VERSIONED:-versioned}" \
-      --listers-package "${OUTPUT_PKG}/listers" \
-      --output-package "${OUTPUT_PKG}/informers" \
-      "$@"
-fi

staging/src/k8s.io/code-generator/generate-internal-groups.sh

@@ -25,17 +25,21 @@ if [ "$#" -lt 5 ] || [ "${1}" == "--help" ]; then
   cat <<EOF
 Usage: $(basename "$0") <generators> <output-package> <internal-apis-package> <extensiona-apis-package> <groups-versions> ...
 
-  <generators>        the generators comma separated to run (deepcopy,defaulter,conversion,client,lister,informer,openapi) or "all".
+  <generators>        the generators comma separated to run (applyconfiguration,client,conversion,deepcopy,defaulter,informer,lister,openapi).
   <output-package>    the output package name (e.g. github.com/example/project/pkg/generated).
-  <int-apis-package>  the internal types dir (e.g. github.com/example/project/pkg/apis).
+  <int-apis-package>  the internal types dir (e.g. github.com/example/project/pkg/apis) or "" if none.
   <ext-apis-package>  the external types dir (e.g. github.com/example/project/pkg/apis or githubcom/example/apis).
   <groups-versions>   the groups and their versions in the format "groupA:v1,v2 groupB:v1 groupC:v2", relative
                       to <api-package>.
   ...                 arbitrary flags passed to all generator binaries.
 
-Examples:
+Example:
-  $(basename "$0") all                           github.com/example/project/pkg/client github.com/example/project/pkg/apis github.com/example/project/pkg/apis "foo:v1 bar:v1alpha1,v1beta1"
+  $(basename "$0") \
-  $(basename "$0") deepcopy,defaulter,conversion github.com/example/project/pkg/client github.com/example/project/pkg/apis github.com/example/project/apis     "foo:v1 bar:v1alpha1,v1beta1"
+      deepcopy,defaulter,conversion \
+      github.com/example/project/pkg/client \
+      github.com/example/project/pkg/apis \
+      github.com/example/project/apis \
+      "foo:v1 bar:v1alpha1,v1beta1"
 EOF
   exit 0
 fi
@@ -47,11 +51,32 @@ EXT_APIS_PKG="$4"
 GROUPS_WITH_VERSIONS="$5"
 shift 5
 
+if [ "${GENS}" = "all" ] || grep -qw "all" <<<"${GENS}"; then
+    ALL="client,conversion,deepcopy,defaulter,informer,lister,openapi"
+    echo "WARNING: Specifying \"all\" as a generator is deprecated."
+    echo "WARNING: Please list the specific generators needed."
+    echo "WARNING: \"all\" is now an alias for \"${ALL}\"; new code generators WILL NOT be added to this set"
+    echo
+    GENS="${ALL}"
+fi
+
 (
   # To support running this script from anywhere, first cd into this directory,
   # and then install with forced module mode on and fully qualified name.
   cd "$(dirname "${0}")"
-  GO111MODULE=on go install k8s.io/code-generator/cmd/{defaulter-gen,conversion-gen,client-gen,lister-gen,informer-gen,deepcopy-gen,openapi-gen}
+  BINS=(
+      applyconfiguration-gen
+      client-gen
+      conversion-gen
+      deepcopy-gen
+      defaulter-gen
+      informer-gen
+      lister-gen
+      openapi-gen
+  )
+  # Compile all the tools at once - it's slightly faster but also just simpler.
+  # shellcheck disable=2046 # printf word-splitting is intentional
+  GO111MODULE=on go install $(printf "k8s.io/code-generator/cmd/%s " "${BINS[@]}")
 )
 
 # Go installs the above commands to get installed in $GOBIN if defined, and $GOPATH/bin otherwise:
@@ -77,38 +102,51 @@ for GVs in ${GROUPS_WITH_VERSIONS}; do
   done
 done
 
-if [ "${GENS}" = "all" ] || grep -qw "deepcopy" <<<"${GENS}"; then
+if grep -qw "deepcopy" <<<"${GENS}"; then
   echo "Generating deepcopy funcs"
   "${gobin}/deepcopy-gen" \
-      --input-dirs "$(codegen::join , "${ALL_FQ_APIS[@]}")" -O zz_generated.deepcopy \
+      --input-dirs "$(codegen::join , "${ALL_FQ_APIS[@]}")" \
+      -O zz_generated.deepcopy \
       "$@"
 fi
 
-if [ "${GENS}" = "all" ] || grep -qw "defaulter" <<<"${GENS}"; then
+if grep -qw "defaulter" <<<"${GENS}"; then
   echo "Generating defaulters"
   "${gobin}/defaulter-gen"  \
-      --input-dirs "$(codegen::join , "${EXT_FQ_APIS[@]}")" -O zz_generated.defaults \
+      --input-dirs "$(codegen::join , "${EXT_FQ_APIS[@]}")" \
+      -O zz_generated.defaults \
       "$@"
 fi
 
-if [ "${GENS}" = "all" ] || grep -qw "conversion" <<<"${GENS}"; then
+if grep -qw "conversion" <<<"${GENS}"; then
   echo "Generating conversions"
   "${gobin}/conversion-gen" \
-      --input-dirs "$(codegen::join , "${ALL_FQ_APIS[@]}")" -O zz_generated.conversion \
+      --input-dirs "$(codegen::join , "${ALL_FQ_APIS[@]}")" \
+      -O zz_generated.conversion \
       "$@"
 fi
 
-if [ "${GENS}" = "all" ] || grep -qw "client" <<<"${GENS}"; then
+if grep -qw "applyconfiguration" <<<"${GENS}"; then
+  echo "Generating apply configuration for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/${APPLYCONFIGURATION_PKG_NAME:-applyconfiguration}"
+  APPLY_CONFIGURATION_PACKAGE="${OUTPUT_PKG}/${APPLYCONFIGURATION_PKG_NAME:-applyconfiguration}"
+  "${gobin}/applyconfiguration-gen" \
+      --input-dirs "$(codegen::join , "${EXT_FQ_APIS[@]}")" \
+      --output-package "${APPLY_CONFIGURATION_PACKAGE}" \
+      "$@"
+fi
+
+if grep -qw "client" <<<"${GENS}"; then
   echo "Generating clientset for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}"
   "${gobin}/client-gen" \
       --clientset-name "${CLIENTSET_NAME_VERSIONED:-versioned}" \
       --input-base "" \
       --input "$(codegen::join , "${EXT_FQ_APIS[@]}")" \
       --output-package "${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}" \
+      --apply-configuration-package "${APPLY_CONFIGURATION_PACKAGE:-}" \
       "$@"
 fi
 
-if [ "${GENS}" = "all" ] || grep -qw "lister" <<<"${GENS}"; then
+if grep -qw "lister" <<<"${GENS}"; then
   echo "Generating listers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/listers"
   "${gobin}/lister-gen" \
       --input-dirs "$(codegen::join , "${EXT_FQ_APIS[@]}")" \
@@ -116,7 +154,7 @@ if [ "${GENS}" = "all" ] || grep -qw "lister" <<<"${GENS}"; then
       "$@"
 fi
 
-if [ "${GENS}" = "all" ] || grep -qw "informer" <<<"${GENS}"; then
+if grep -qw "informer" <<<"${GENS}"; then
   echo "Generating informers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/informers"
   "${gobin}/informer-gen" \
       --input-dirs "$(codegen::join , "${EXT_FQ_APIS[@]}")" \
@@ -126,7 +164,7 @@ if [ "${GENS}" = "all" ] || grep -qw "informer" <<<"${GENS}"; then
       "$@"
 fi
 
-if [ "${GENS}" = "all" ] || grep -qw "openapi" <<<"${GENS}"; then
+if grep -qw "openapi" <<<"${GENS}"; then
   echo "Generating OpenAPI definitions for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/openapi"
   declare -a OPENAPI_EXTRA_PACKAGES
   "${gobin}/openapi-gen" \

staging/src/k8s.io/code-generator/hack/update-codegen.sh

@@ -24,26 +24,26 @@ SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 # - --output-base because this script should also be able to run inside the vendor dir of
 #   k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir
 #   instead of the $GOPATH directly. For normal projects this can be dropped.
-"${SCRIPT_ROOT}/generate-internal-groups.sh" all \
+"${SCRIPT_ROOT}/generate-internal-groups.sh" "client,conversion,deepcopy,defaulter,informer,lister,openapi" \
   k8s.io/code-generator/examples/apiserver \
   k8s.io/code-generator/examples/apiserver/apis \
   k8s.io/code-generator/examples/apiserver/apis \
   "example:v1 example2:v1 example3.io:v1" \
   --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
   --go-header-file "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
-"${SCRIPT_ROOT}/generate-groups.sh" all \
+"${SCRIPT_ROOT}/generate-groups.sh" "applyconfiguration,client,deepcopy,defaulter,informer,lister" \
   k8s.io/code-generator/examples/crd \
   k8s.io/code-generator/examples/crd/apis \
   "example:v1 example2:v1" \
   --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
   --go-header-file "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
-"${SCRIPT_ROOT}/generate-groups.sh" all \
+"${SCRIPT_ROOT}/generate-groups.sh" "applyconfiguration,client,deepcopy,defaulter,informer,lister" \
   k8s.io/code-generator/examples/MixedCase \
   k8s.io/code-generator/examples/MixedCase/apis \
   "example:v1" \
   --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
   --go-header-file "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
-"${SCRIPT_ROOT}/generate-groups.sh" all \
+"${SCRIPT_ROOT}/generate-groups.sh" "applyconfiguration,client,deepcopy,defaulter,informer,lister" \
   k8s.io/code-generator/examples/HyphenGroup \
   k8s.io/code-generator/examples/HyphenGroup/apis \
   "example:v1" \

staging/src/k8s.io/kube-aggregator/hack/update-codegen.sh

@@ -23,7 +23,7 @@ CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-
 
 CLIENTSET_NAME_VERSIONED=clientset \
 CLIENTSET_PKG_NAME=clientset_generated \
-"${CODEGEN_PKG}/generate-groups.sh" "deepcopy,client,lister,informer" \
+"${CODEGEN_PKG}/generate-groups.sh" "deepcopy,defaulter,client,lister,informer" \
   k8s.io/kube-aggregator/pkg/client \
   k8s.io/kube-aggregator/pkg/apis \
   "apiregistration:v1beta1,v1" \

staging/src/k8s.io/metrics/hack/update-codegen.sh

@@ -32,7 +32,7 @@ CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-
   k8s.io/metrics/pkg/client \
   k8s.io/metrics/pkg/apis \
   k8s.io/metrics/pkg/apis \
-  "metrics:v1alpha1,v1beta1 custom_metrics:v1beta1 external_metrics:v1beta1" \
+  "metrics:v1alpha1,v1beta1 custom_metrics:v1beta1,v1beta2 external_metrics:v1beta1" \
   --output-base "$(dirname "${BASH_SOURCE[0]}")/../../.." \
   --go-header-file "${SCRIPT_ROOT}/hack/boilerplate.go.txt"
 "${CODEGEN_PKG}/generate-groups.sh" "client" \

staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh

@@ -25,7 +25,7 @@ CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-
 # --output-base    because this script should also be able to run inside the vendor dir of
 #                  k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir
 #                  instead of the $GOPATH directly. For normal projects this can be dropped.
-"${CODEGEN_PKG}/generate-groups.sh" all \
+"${CODEGEN_PKG}/generate-groups.sh" "applyconfiguration,client,deepcopy,informer,lister" \
   k8s.io/sample-apiserver/pkg/generated \
   k8s.io/sample-apiserver/pkg/apis \
   "wardle:v1alpha1,v1beta1" \