github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #44073 from marun/fed-e2e-config-from-secrets

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 43500, 44073)

[Federation] Add option to retrieve e2e cluster config from secrets

Previously the federation e2e setup was reading member cluster configuration from the test run's kubeconfig. This change removes that dependency in favor of reading member cluster configuration from secrets in the hosting cluster, and caches the configuration to avoid having to read it separately for each test.

cc: @kubernetes/sig-federation-pr-reviews @perotinus
This commit is contained in:
Kubernetes Submit Queue 2017-04-18 22:27:58 -07:00 committed by GitHub
commit fe44d1f5ce
6 changed files with 126 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hack/verify-flags/known-flags.txt
View File

@ -268,6 +268,7 @@ feature-gates
federated-api-burst
federated-api-qps
federated-kube-context
federation-config-from-cluster
federation-name
federation-system-namespace
federation-upgrade-target

3
test/e2e/framework/test_context.go
View File

@ -92,6 +92,8 @@ type TestContextType struct {
FederatedKubeContext string
// Federation control plane version to upgrade to while doing upgrade tests
FederationUpgradeTarget string
// Whether configuration for accessing federation member clusters should be sourced from the host cluster
FederationConfigFromCluster bool
// Viper-only parameters. These will in time replace all flags.
@ -179,6 +181,7 @@ func RegisterClusterFlags() {
flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver")
flag.StringVar(&TestContext.FederatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.")
flag.BoolVar(&TestContext.FederationConfigFromCluster, "federation-config-from-cluster", false, "whether to source configuration for member clusters from the hosting cluster.")
flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")

2
test/e2e_federation/apiserver.go
View File

@ -57,7 +57,7 @@ var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func
framework.Logf("Checking that %d clusters are Ready", len(contexts))
for _, context := range contexts {
fedframework.ClusterIsReadyOrFail(f, &context)
fedframework.ClusterIsReadyOrFail(f, context.Name)
}
framework.Logf("%d clusters are Ready", len(contexts))

2
test/e2e_federation/framework/BUILD
View File

@ -21,6 +21,7 @@ go_library(
"//federation/client/clientset_generated/federation_clientset:go_default_library",
"//federation/pkg/federatedtypes:go_default_library",
"//federation/pkg/federatedtypes/crudtester:go_default_library",
"//federation/pkg/federation-controller/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/api/validation:go_default_library",
@ -35,7 +36,6 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
],
)

159
test/e2e_federation/framework/cluster.go
View File

@ -20,13 +20,13 @@ import (
"fmt"
"time"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
"k8s.io/kubernetes/federation/pkg/federation-controller/util"
"k8s.io/kubernetes/pkg/api/v1"
kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework"
@ -35,15 +35,7 @@ import (
. "github.com/onsi/gomega"
)
const (
KubeAPIQPS float32 = 20.0
KubeAPIBurst = 30
userAgentName = "federation-e2e"
federatedNamespaceTimeout = 5 * time.Minute
federatedClustersWaitTimeout = 1 * time.Minute
)
const federatedClustersWaitTimeout = 1 * time.Minute
// ClusterSlice is a slice of clusters
type ClusterSlice []*Cluster
@ -54,7 +46,8 @@ type Cluster struct {
*kubeclientset.Clientset
}
func getRegisteredClusters(f *Framework) ClusterSlice {
// registeredClustersFromConfig configures clientsets for registered clusters from the e2e kubeconfig
func registeredClustersFromConfig(f *Framework) ClusterSlice {
contexts := f.GetUnderlyingFederatedContexts()
By("Obtaining a list of all the clusters")
@ -62,7 +55,8 @@ func getRegisteredClusters(f *Framework) ClusterSlice {
framework.Logf("Checking that %d clusters are Ready", len(contexts))
for _, context := range contexts {
ClusterIsReadyOrFail(f, &context)
ClusterIsReadyOrFail(f, context.Name)
}
framework.Logf("%d clusters are Ready", len(contexts))
@ -70,12 +64,14 @@ func getRegisteredClusters(f *Framework) ClusterSlice {
for i, c := range clusterList.Items {
framework.Logf("Creating a clientset for the cluster %s", c.Name)
Expect(framework.TestContext.KubeConfig).ToNot(Equal(""), "KubeConfig must be specified to load clusters' client config")
config := restConfigFromContext(c, i)
clusters = append(clusters, &Cluster{
Name: c.Name,
Clientset: createClientsetForCluster(c, i, userAgentName),
Clientset: clientsetFromConfig(f, config, c.Spec.ServerAddressByClientCIDRs[0].ServerAddress),
})
}
waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name, federatedNamespaceTimeout)
waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name)
return clusters
}
@ -100,32 +96,34 @@ func waitForAllRegisteredClusters(f *Framework, clusterCount int) *federationapi
return clusterList
}
func createClientsetForCluster(c federationapi.Cluster, i int, userAgentName string) *kubeclientset.Clientset {
func restConfigFromContext(c federationapi.Cluster, i int) *restclient.Config {
kubecfg, err := clientcmd.LoadFromFile(framework.TestContext.KubeConfig)
framework.ExpectNoError(err, "error loading KubeConfig: %v", err)
cfgOverride := &clientcmd.ConfigOverrides{
ClusterInfo: clientcmdapi.Cluster{
Server: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress,
},
}
ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, cfgOverride, clientcmd.NewDefaultClientConfigLoadingRules())
ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, &clientcmd.ConfigOverrides{}, clientcmd.NewDefaultClientConfigLoadingRules())
cfg, err := ccfg.ClientConfig()
framework.ExpectNoError(err, "Error creating client config in cluster #%d (%q)", i, c.Name)
return cfg
}
cfg.QPS = KubeAPIQPS
cfg.Burst = KubeAPIBurst
return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, userAgentName))
func clientsetFromConfig(f *Framework, cfg *restclient.Config, host string) *kubeclientset.Clientset {
cfg.Host = host
cfg.QPS = f.Framework.Options.ClientQPS
cfg.Burst = f.Framework.Options.ClientBurst
return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, "federation-e2e"))
}
// waitForNamespaceInFederatedClusters waits for the federated namespace to be created in federated clusters
func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string, timeout time.Duration) {
func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string) {
for _, c := range clusters {
name := c.Name
err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q", nsName, name))
err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) {
_, err := c.Clientset.Core().Namespaces().Get(nsName, metav1.GetOptions{})
if err != nil {
By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q, err: %v", nsName, name, err))
if errors.IsNotFound(err) {
return false, nil
} else if err != nil {
framework.Logf("An error occurred waiting for namespace %q to be created in cluster %q: %v", nsName, name, err)
return false, nil
}
By(fmt.Sprintf("Namespace %q exists in cluster %q", nsName, name))
@ -135,22 +133,11 @@ func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string, t
}
}
// ClusterIsReadyOrFail checks whether the federated cluster of the provided context is ready
func ClusterIsReadyOrFail(f *Framework, context *E2EContext) {
c, err := f.FederationClientset.Federation().Clusters().Get(context.Name, metav1.GetOptions{})
framework.ExpectNoError(err, fmt.Sprintf("get cluster: %+v", err))
if c.ObjectMeta.Name != context.Name {
framework.Failf("cluster name does not match input context: actual=%+v, expected=%+v", c, context)
}
err = isReady(context.Name, f.FederationClientset)
framework.ExpectNoError(err, fmt.Sprintf("unexpected error in verifying if cluster %s is ready: %+v", context.Name, err))
framework.Logf("Cluster %s is Ready", context.Name)
}
// Verify that the cluster is marked ready.
func isReady(clusterName string, clientset *fedclientset.Clientset) error {
return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) {
c, err := clientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
// ClusterIsReadyOrFail checks whether the named cluster is ready
func ClusterIsReadyOrFail(f *Framework, clusterName string) {
By(fmt.Sprintf("Checking readiness of cluster %q", clusterName))
err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) {
c, err := f.FederationClientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
if err != nil {
return false, err
}
@ -161,4 +148,86 @@ func isReady(clusterName string, clientset *fedclientset.Clientset) error {
}
return false, nil
})
framework.ExpectNoError(err, fmt.Sprintf("Unexpected error in verifying if cluster %q is ready: %+v", clusterName, err))
framework.Logf("Cluster %s is Ready", clusterName)
}
// Cache the cluster config to avoid having to retrieve it for each test
type clusterConfig struct {
name string
host string
config []byte
}
var cachedClusterConfigs []*clusterConfig
// registeredClustersFromSecrets configures clientsets for cluster access from secrets in the host cluster
func registeredClustersFromSecrets(f *Framework) ClusterSlice {
if cachedClusterConfigs == nil {
cachedClusterConfigs = clusterConfigFromSecrets(f)
}
clusters := ClusterSlice{}
for _, clusterConf := range cachedClusterConfigs {
restConfig := restConfigForCluster(clusterConf)
clientset := clientsetFromConfig(f, restConfig, clusterConf.host)
clusters = append(clusters, &Cluster{
Name: clusterConf.name,
Clientset: clientset,
})
}
waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name)
return clusters
}
// clusterConfigFromSecrets retrieves cluster configuration from
// secrets in the host cluster
func clusterConfigFromSecrets(f *Framework) []*clusterConfig {
By("Obtaining a list of registered clusters")
clusterList, err := f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Error retrieving list of federated clusters: %+v", err))
if len(clusterList.Items) == 0 {
framework.Failf("No registered clusters found")
}
clusterConfigs := []*clusterConfig{}
for _, c := range clusterList.Items {
ClusterIsReadyOrFail(f, c.Name)
config := clusterConfigFromSecret(f, c.Name, c.Spec.SecretRef.Name)
clusterConfigs = append(clusterConfigs, &clusterConfig{
name: c.Name,
host: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress,
config: config,
})
}
return clusterConfigs
}
// clusterConfigFromSecret retrieves configuration for a accessing a
// cluster from a secret in the host cluster
func clusterConfigFromSecret(f *Framework, clusterName string, secretName string) []byte {
By(fmt.Sprintf("Loading configuration for cluster %q", clusterName))
namespace := framework.FederationSystemNamespace()
secret, err := f.Framework.ClientSet.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Error loading config secret \"%s/%s\" for cluster %q: %+v", namespace, secretName, clusterName, err))
config, ok := secret.Data[util.KubeconfigSecretDataKey]
if !ok || len(config) == 0 {
framework.Failf("Secret \"%s/%s\" for cluster %q has no value for key %q", namespace, secretName, clusterName, util.KubeconfigSecretDataKey)
}
return config
}
// restConfigForCluster creates a rest client config for the given cluster config
func restConfigForCluster(clusterConf *clusterConfig) *restclient.Config {
cfg, err := clientcmd.Load(clusterConf.config)
framework.ExpectNoError(err, fmt.Sprintf("Error loading configuration for cluster %q: %+v", clusterConf.name, err))
restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig()
framework.ExpectNoError(err, fmt.Sprintf("Error creating client for cluster %q: %+v", clusterConf.name, err))
return restConfig
}

8
test/e2e_federation/framework/framework.go
View File

@ -228,11 +228,15 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext {
}
func (f *Framework) GetRegisteredClusters() ClusterSlice {
return getRegisteredClusters(f)
if framework.TestContext.FederationConfigFromCluster {
return registeredClustersFromSecrets(f)
} else {
return registeredClustersFromConfig(f)
}
}
func (f *Framework) GetClusterClients() []kubeclientset.Interface {
clusters := getRegisteredClusters(f)
clusters := f.GetRegisteredClusters()
var clusterClients []kubeclientset.Interface
for _, c := range clusters {
clusterClients = append(clusterClients, c.Clientset)