Check if default config is invalid before comparing it

Changes made in #31947 cause kube-proxy to ignore `--kubeconfig`
and `--master` flags and use in-cluster configuration, which is
unusable due to the fact that VIP hasn't been created yet.

pkg/client/unversioned/clientcmd/merged_client_builder.go

@@ -113,6 +113,9 @@ func (config *DeferredLoadingClientConfig) ClientConfig() (*restclient.Config, e
 		// TODO: this shouldn't be a global - the client config rules should be
 		//   handling this.
 		defaultConfig, err := DefaultClientConfig.ClientConfig()
+		if IsConfigurationInvalid(err) {
+			return mergedConfig, nil
+		}
 		if err == nil && !reflect.DeepEqual(mergedConfig, defaultConfig) {
 			return mergedConfig, nil
 		}