Added support for image credential provider for windows and arm64 on gce

cluster/gce/config-common.sh

@@ -167,3 +167,17 @@ export CSI_PROXY_STORAGE_PATH="https://storage.googleapis.com/gke-release/csi-pr
 export CSI_PROXY_VERSION="${CSI_PROXY_VERSION:-v1.1.1-gke.0}"
 # csi-proxy additional flags, there are additional flags that cannot be unset in k8s-node-setup.psm1
 export CSI_PROXY_FLAGS="${CSI_PROXY_FLAGS:-}"
+# Storage path for auth-provider-gcp binaries
+export AUTH_PROVIDER_GCP_STORAGE_PATH="${AUTH_PROVIDER_GCP_STORAGE_PATH:-https://storage.googleapis.com/gke-release/auth-provider-gcp}"
+# auth-provider-gcp version
+export AUTH_PROVIDER_GCP_VERSION="${AUTH_PROVIDER_GCP_VERSION:-v0.0.2-gke.4}"
+# Hash of auth-provider-gcp.exe binary
+export AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64="${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64:-348af2c189d938e1a4fa5ac5c640d21e003da1f000abcd6fd7eef2acd0678638286e40703618758d4fdfe2cc4b90e920f0422128ec777c74054af9dd4405de12}"
+# Directory of kubelet image credential provider binary files on windows
+export AUTH_PROVIDER_GCP_LINUX_BIN_DIR="${AUTH_PROVIDER_GCP_LINUX_BIN_DIR:-/home/kubernetes/bin}"
+# Location of kubelet image credential provider config file on windows
+export AUTH_PROVIDER_GCP_LINUX_CONF_FILE="${AUTH_PROVIDER_GCP_LINUX_CONF_FILE:-/home/kubernetes/cri-auth-config.yaml}"
+# Directory of kubelet image credential provider binary files on windows
+export AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR:-${WINDOWS_NODE_DIR}}
+# Location of kubelet image credential provider config file on windows
+export AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE="${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE:-${WINDOWS_K8S_DIR}\cri-auth-config.yaml}"

cluster/gce/gci/configure.sh

@@ -28,14 +28,14 @@ DEFAULT_CNI_VERSION='v1.2.0'
 DEFAULT_CNI_HASH='29ea9be8e81e0b4c44469c4307cd8be83647e30ade8b737d94df81477b494662308b2566fce80cfa993c761afb6e5bad9382455260b857c7f941fa18bb7919b4'
 DEFAULT_NPD_VERSION='v0.8.9'
 DEFAULT_NPD_HASH_AMD64='4919c47447c5f3871c1dc3171bbb817a38c8c8d07a6ce55a77d43cadc098e9ad608ceeab121eec00c13c0b6a2cc3488544d61ce84cdade1823f3fd5163a952de'
-DEFAULT_AUTH_PROVIDER_GCP_HASH_AMD64='88d9fa581002973170ca58427763f00355b24fbabd66f7fee725a0845ad88bee644e60eed2d95a5721e6ae0056a81a5990bf02148ea49817c174bcb2cc9c0626'
-DEFAULT_AUTH_PROVIDER_GCP_VERSION='v0.24.0'
 # TODO (SergeyKanzhelev): fill up for npd 0.8.9+
 DEFAULT_NPD_HASH_ARM64='8ccb42a862efdfc1f25ca9a22f3fd36f9fdff1ac618dd7d39e3b5991505dd610d432364420896ad71f42197a116f28a85dde58b129baa075ebb7312caa57f852'
 DEFAULT_CRICTL_VERSION='v1.26.1'
 DEFAULT_CRICTL_AMD64_SHA512='e3a20c4d18bbbd9f5dc303e90b649181f9b8242758de58d04ea8acd0d3da32919b8859e90b6602354755dc82b79833caf30510e5a27b0ebc6a2b1545d744d4cc'
 DEFAULT_CRICTL_ARM64_SHA512='605d8fe73e7e0b93dfac9ea90548e7334b55dda1f0abeb4c3382ae15d6d250a71fccfde20e8fd68bde59b4c1d54ec69ef295a2aa2119e8579d60d0dbcf380d2e'
 DEFAULT_MOUNTER_TAR_SHA='7956fd42523de6b3107ddc3ce0e75233d2fcb78436ff07a1389b6eaac91fb2b1b72a08f7a219eaf96ba1ca4da8d45271002e0d60e0644e796c665f99bb356516'
+AUTH_PROVIDER_GCP_HASH_LINUX_AMD64="${AUTH_PROVIDER_GCP_HASH_LINUX_AMD64:-156058e5b3994cba91c23831774033e0d505d6d8b80f43541ef6af91b320fd9dfaabe42ec8a8887b51d87104c2b57e1eb895649d681575ffc80dd9aee8e563db}"
+AUTH_PROVIDER_GCP_HASH_LINUX_ARM64="${AUTH_PROVIDER_GCP_HASH_LINUX_ARM64:-1aa3b0bea10a9755231989ffc150cbfa770f1d96932db7535473f7bfeb1108bafdae80202ae738d59495982512e716ff7366d5f414d0e76dd50519f98611f9ab}"
 ###
 
 # Standard curl flags.
@@ -549,30 +549,33 @@ function install-containerd-ubuntu {
 }
 
 function install-auth-provider-gcp {
-  local -r auth_provider_tar="auth-provider-gcp-${DEFAULT_AUTH_PROVIDER_GCP_VERSION}-${HOST_PLATFORM}_${HOST_ARCH}.tar.gz"
+  local -r filename="auth-provider-gcp"
-  echo "Downloading auth-provider-gcp ${auth_provider_tar}" .
+  local -r auth_provider_storage_full_path="${AUTH_PROVIDER_GCP_STORAGE_PATH}/${AUTH_PROVIDER_GCP_VERSION}/${HOST_PLATFORM}_${HOST_ARCH}/${filename}"
+  echo "Downloading auth-provider-gcp ${auth_provider_storage_full_path}" .
 
-  local -r auth_provider_release_path="https://storage.googleapis.com/cloud-provider-gcp"
+  case "${HOST_ARCH}" in
-  download-or-bust "${DEFAULT_AUTH_PROVIDER_GCP_HASH_AMD64}" "${auth_provider_release_path}/${auth_provider_tar}"
+    amd64)
+      local -r auth_provider_gcp_hash="${AUTH_PROVIDER_GCP_HASH_LINUX_AMD64}"
+      ;;
+    arm64)
+      local -r auth_provider_gcp_hash="${AUTH_PROVIDER_GCP_HASH_LINUX_ARM64}"
+      ;;
+    *)
+      echo "Unrecognized version and platform/arch combination: ${HOST_PLATFORM}/${HOST_ARCH}"
+      exit 1
+  esac
 
-  # Keep in sync with --image-credential-provider-bin-dir in ../util.sh
+  download-or-bust "${auth_provider_gcp_hash}" "${auth_provider_storage_full_path}"
-  local auth_provider_dir="${KUBE_HOME}/auth-provider-gcp"
-  mkdir -p "${auth_provider_dir}"
-  tar xzf "${KUBE_HOME}/${auth_provider_tar}" -C "${auth_provider_dir}" --overwrite
-  mv "${auth_provider_dir}/auth-provider-gcp" "${KUBE_BIN}"
-  chmod a+x "${KUBE_BIN}/auth-provider-gcp"
 
-  rm -f "${KUBE_HOME}/${auth_provider_tar}"
+  mv "${KUBE_HOME}/${filename}" "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}"
-  rmdir "${auth_provider_dir}"
+  chmod a+x "${KUBE_BIN}/${filename}"
 
-  # Keep in sync with --image-credential-provider-config in ../util.sh
+  cat >> "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}" << EOF
-  local auth_config_file="${KUBE_HOME}/cri_auth_config.yaml"
-  cat >> "${auth_config_file}" << EOF
 kind: CredentialProviderConfig
-apiVersion: kubelet.config.k8s.io/v1beta1
+apiVersion: kubelet.config.k8s.io/v1
 providers:
   - name: auth-provider-gcp
-    apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
     matchImages:
     - "container.cloud.google.com"
     - "gcr.io"

cluster/gce/util.sh

@@ -758,8 +758,8 @@ function construct-linux-kubelet-flags {
     # Keep the values of --image-credential-provider-config and --image-credential-provider-bin-dir
     # in sync with value of auth_config_file and auth_provider_dir set in install-auth-provider-gcp function
     # in gci/configure.sh.
-    flags+="  --image-credential-provider-config=/home/kubernetes/cri_auth_config.yaml"
+    flags+="  --image-credential-provider-config=${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}"
-    flags+="  --image-credential-provider-bin-dir=/home/kubernetes/bin"
+    flags+="  --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}"
   fi
 
   if [[ "${node_type}" == "master" ]]; then
@@ -864,6 +864,13 @@ function construct-windows-kubelet-flags {
   WINDOWS_CONTAINER_RUNTIME_ENDPOINT=${KUBE_WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd}
   flags+=" --container-runtime-endpoint=${WINDOWS_CONTAINER_RUNTIME_ENDPOINT}"
 
+  # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth
+  # credential provider. https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider
+  if [[ "${ENABLE_AUTH_PROVIDER_GCP:-false}" == "true" ]]; then
+    flags+="  --image-credential-provider-config=${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}"
+    flags+="  --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}"
+  fi
+
   KUBELET_ARGS="${flags}"
 }
 
@@ -1199,6 +1206,10 @@ ${CUSTOM_CALICO_NODE_DAEMONSET_YAML//\'/\'\'}
 CUSTOM_TYPHA_DEPLOYMENT_YAML: |
 ${CUSTOM_TYPHA_DEPLOYMENT_YAML//\'/\'\'}
 CONCURRENT_SERVICE_SYNCS: $(yaml-quote "${CONCURRENT_SERVICE_SYNCS:-}")
+AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
+AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
+AUTH_PROVIDER_GCP_LINUX_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}")
+AUTH_PROVIDER_GCP_LINUX_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}")
 EOF
   if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \
      [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]]  || \
@@ -1581,6 +1592,11 @@ NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEAS
 NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS}")
 NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}")
 WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE}")
+AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
+AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
+AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64: $(yaml-quote "${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64}")
+AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}")
+AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}")
 EOF
 }
 

cluster/gce/windows/configure.ps1

@@ -165,11 +165,13 @@ try {
   DownloadAndInstall-KubernetesBinaries
   DownloadAndInstall-NodeProblemDetector
   DownloadAndInstall-CSIProxyBinaries
+  DownloadAndInstall-AuthProviderGcpBinary
   Start-CSIProxy
   Create-NodePki
   Create-KubeletKubeconfig
   Create-KubeproxyKubeconfig
   Create-NodeProblemDetectorKubeConfig
+  Create-AuthProviderGcpConfig
   Set-PodCidr
   Configure-HostNetworkingService
   Prepare-CniNetworking

cluster/gce/windows/k8s-node-setup.psm1

@@ -297,6 +297,12 @@ function Set-EnvironmentVars {
     "WINDOWS_ENABLE_HYPERV" = ${kube_env}['WINDOWS_ENABLE_HYPERV']
     "ENABLE_NODE_PROBLEM_DETECTOR" = ${kube_env}['ENABLE_NODE_PROBLEM_DETECTOR']
     "NODEPROBLEMDETECTOR_KUBECONFIG_FILE" = ${kube_env}['WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE']
+    "ENABLE_AUTH_PROVIDER_GCP" = ${kube_env}['ENABLE_AUTH_PROVIDER_GCP']
+    "AUTH_PROVIDER_GCP_STORAGE_PATH" = ${kube_env}['AUTH_PROVIDER_GCP_STORAGE_PATH']
+    "AUTH_PROVIDER_GCP_VERSION" = ${kube_env}['AUTH_PROVIDER_GCP_VERSION']
+    "AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64" = ${kube_env}['AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64']
+    "AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR" = ${kube_env}['AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR']
+    "AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE" = ${kube_env}['AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE']
 
     "Path" = ${env:Path} + ";" + ${kube_env}['NODE_DIR']
     "KUBE_NETWORK" = "l2bridge".ToLower()
@@ -2286,5 +2292,51 @@ $FLUENTD_CONFIG = @'
 </filter>
 '@
 
+# Downloads the out-of-tree kubelet image credential provider binaries.
+function DownloadAndInstall-AuthProviderGcpBinary {
+  if ("${env:ENABLE_AUTH_PROVIDER_GCP}" -eq "true") {
+    $filename = 'auth-provider-gcp.exe'
+    if (ShouldWrite-File ${env:AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}\$filename) {
+      Log-Output "Installing auth provider gcp binaries"
+      $tmp_dir = 'C:\k8s_tmp'
+      New-Item -Force -ItemType 'directory' $tmp_dir | Out-Null
+      $url = "${env:AUTH_PROVIDER_GCP_STORAGE_PATH}/${env:AUTH_PROVIDER_GCP_VERSION}/windows_amd64/$filename"
+      MustDownload-File -Hash $AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64 -Algorithm SHA512 -OutFile $tmp_dir\$filename -URLs $url
+      Move-Item -Force $tmp_dir\$filename ${env:AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}
+      Remove-Item -Force -Recurse $tmp_dir
+    } else {
+      Log-Output "Skipping auth provider gcp binaries installation, auth-provider-gcp.exe file already exists."
+    }
+  }
+}
+
+# Creates config file for the out-of-tree kubelet image credential provider.
+function Create-AuthProviderGcpConfig {
+  if ("${env:ENABLE_AUTH_PROVIDER_GCP}" -eq "true") {
+    if (ShouldWrite-File ${env:AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}) {
+      Log-Output "Creating auth provider gcp config file"
+      Set-Content ${env:AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE} @'
+kind: CredentialProviderConfig
+apiVersion: kubelet.config.k8s.io/v1
+providers:
+  - name: auth-provider-gcp.exe
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    matchImages:
+    - "container.cloud.google.com"
+    - "gcr.io"
+    - "*.gcr.io"
+    - "*.pkg.dev"
+    args:
+    - get-credentials
+    - --v=3
+    defaultCacheDuration: 1m
+'@
+    } else {
+      Log-Output "Skipping auth provider gcp config file creation, it already exists"
+    }
+  }
+}
+
+
 # Export all public functions:
 Export-ModuleMember -Function *-*