diff --git a/test/e2e/common/util.go b/test/e2e/common/util.go index 9db11885b5f..e687067c279 100644 --- a/test/e2e/common/util.go +++ b/test/e2e/common/util.go @@ -23,8 +23,9 @@ import ( type Suite string const ( - E2E Suite = "e2e" - NodeE2E Suite = "node e2e" + E2E Suite = "e2e" + NodeE2E Suite = "node e2e" + FederationE2E Suite = "federation e2e" ) var CurrentSuite Suite diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 687d1e61477..14d9fb6ed4a 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -42,6 +42,7 @@ import ( commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/generated" + federationtest "k8s.io/kubernetes/test/e2e_federation" testutils "k8s.io/kubernetes/test/utils" ) @@ -153,6 +154,9 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { // Reference common test to make the import valid. commontest.CurrentSuite = commontest.E2E + // Reference federation test to make the import valid. + federationtest.FederationSuite = commontest.FederationE2E + return nil }, func(data []byte) { diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 509c3bccf89..6d4244527c2 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -20,7 +20,6 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" "reflect" "strings" "sync" @@ -34,7 +33,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" staging "k8s.io/client-go/kubernetes" clientreporestclient "k8s.io/client-go/rest" - "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" @@ -48,7 +46,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - yaml "gopkg.in/yaml.v2" ) const ( @@ -88,13 +85,6 @@ type Framework struct { // configuration for framework's client options FrameworkOptions - - // will this framework exercise a federated cluster as well - federated bool - - // Federation specific params. These are set only if federated = true. - FederationClientset_1_5 *federation_clientset.Clientset - FederationNamespace *v1.Namespace } type TestDataSummary interface { @@ -118,12 +108,6 @@ func NewDefaultFramework(baseName string) *Framework { return NewFramework(baseName, options, nil) } -func NewDefaultFederatedFramework(baseName string) *Framework { - f := NewDefaultFramework(baseName) - f.federated = true - return f -} - func NewDefaultGroupVersionFramework(baseName string, groupVersion schema.GroupVersion) *Framework { f := NewDefaultFramework(baseName) f.options.GroupVersion = &groupVersion @@ -203,25 +187,6 @@ func (f *Framework) BeforeEach() { f.ClientPool = dynamic.NewClientPool(config, api.Registry.RESTMapper(), dynamic.LegacyAPIPathResolverFunc) } - if f.federated { - if f.FederationClientset_1_5 == nil { - By("Creating a release 1.4 federation Clientset") - var err error - f.FederationClientset_1_5, err = LoadFederationClientset_1_5() - Expect(err).NotTo(HaveOccurred()) - } - By("Waiting for federation-apiserver to be ready") - err := WaitForFederationApiserverReady(f.FederationClientset_1_5) - Expect(err).NotTo(HaveOccurred()) - By("federation-apiserver is ready") - - By("Creating a federation namespace") - ns, err := f.createFederationNamespace(f.BaseName) - Expect(err).NotTo(HaveOccurred()) - f.FederationNamespace = ns - By(fmt.Sprintf("Created federation namespace %s", ns.Name)) - } - By("Building a namespace api object") namespace, err := f.CreateNamespace(f.BaseName, map[string]string{ "e2e-framework": f.BaseName, @@ -262,45 +227,6 @@ func (f *Framework) BeforeEach() { } } -func (f *Framework) deleteFederationNs() { - if !f.federated { - // Nothing to do if this is not a federation setup. - return - } - ns := f.FederationNamespace - By(fmt.Sprintf("Destroying federation namespace %q for this suite.", ns.Name)) - timeout := 5 * time.Minute - if f.NamespaceDeletionTimeout != 0 { - timeout = f.NamespaceDeletionTimeout - } - - clientset := f.FederationClientset_1_5 - // First delete the namespace from federation apiserver. - // Also delete the corresponding namespaces from underlying clusters. - orphanDependents := false - if err := clientset.Core().Namespaces().Delete(ns.Name, &v1.DeleteOptions{OrphanDependents: &orphanDependents}); err != nil { - Failf("Error while deleting federation namespace %s: %s", ns.Name, err) - } - // Verify that it got deleted. - err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { - if _, err := clientset.Core().Namespaces().Get(ns.Name, metav1.GetOptions{}); err != nil { - if apierrors.IsNotFound(err) { - return true, nil - } - Logf("Error while waiting for namespace to be terminated: %v", err) - return false, nil - } - return false, nil - }) - if err != nil { - if !apierrors.IsNotFound(err) { - Failf("Couldn't delete ns %q: %s", ns.Name, err) - } else { - Logf("Namespace %v was already deleted", ns.Name) - } - } -} - // AfterEach deletes the namespace, after reading its events. func (f *Framework) AfterEach() { RemoveCleanupAction(f.cleanupHandle) @@ -327,8 +253,6 @@ func (f *Framework) AfterEach() { } } } - // Delete the federation namespace. - f.deleteFederationNs() } else { if TestContext.DeleteNamespace { Logf("Found DeleteNamespace=false, skipping namespace deletion!") @@ -340,7 +264,6 @@ func (f *Framework) AfterEach() { // Paranoia-- prevent reuse! f.Namespace = nil - f.FederationNamespace = nil f.ClientSet = nil f.namespacesToDelete = nil @@ -354,34 +277,12 @@ func (f *Framework) AfterEach() { } }() - if f.federated { - defer func() { - if f.FederationClientset_1_5 == nil { - Logf("Warning: framework is marked federated, but has no federation 1.4 clientset") - return - } - if err := f.FederationClientset_1_5.Federation().Clusters().DeleteCollection(nil, v1.ListOptions{}); err != nil { - Logf("Error: failed to delete Clusters: %+v", err) - } - }() - } - // Print events if the test failed. if CurrentGinkgoTestDescription().Failed && TestContext.DumpLogsOnFailure { // Pass both unversioned client and and versioned clientset, till we have removed all uses of the unversioned client. DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name) By(fmt.Sprintf("Dumping a list of prepulled images on each node")) LogContainersInPodsWithLabels(f.ClientSet, api.NamespaceSystem, ImagePullerLabels, "image-puller", Logf) - if f.federated { - // Dump federation events in federation namespace. - DumpEventsInNamespace(func(opts v1.ListOptions, ns string) (*v1.EventList, error) { - return f.FederationClientset_1_5.Core().Events(ns).List(opts) - }, f.FederationNamespace.Name) - // Print logs of federation control plane pods (federation-apiserver and federation-controller-manager) - LogPodsWithLabels(f.ClientSet, "federation", map[string]string{"app": "federated-cluster"}, Logf) - // Print logs of kube-dns pod - LogPodsWithLabels(f.ClientSet, "kube-system", map[string]string{"k8s-app": "kube-dns"}, Logf) - } } summaries := make([]TestDataSummary, 0) @@ -456,29 +357,6 @@ func (f *Framework) CreateNamespace(baseName string, labels map[string]string) ( return ns, err } -func (f *Framework) createFederationNamespace(baseName string) (*v1.Namespace, error) { - clientset := f.FederationClientset_1_5 - namespaceObj := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("e2e-tests-%v-", baseName), - }, - } - // Be robust about making the namespace creation call. - var got *v1.Namespace - if err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) { - var err error - got, err = clientset.Core().Namespaces().Create(namespaceObj) - if err != nil { - Logf("Unexpected error while creating namespace: %v", err) - return false, nil - } - return true, nil - }); err != nil { - return nil, err - } - return got, nil -} - // WaitForPodTerminated waits for the pod to be terminated with the given reason. func (f *Framework) WaitForPodTerminated(podName, reason string) error { return waitForPodTerminatedInNamespace(f.ClientSet, podName, reason, f.Namespace.Name) @@ -709,7 +587,7 @@ type KubeConfig struct { Users []KubeUser `yaml:"users"` } -func (kc *KubeConfig) findUser(name string) *KubeUser { +func (kc *KubeConfig) FindUser(name string) *KubeUser { for _, user := range kc.Users { if user.Name == name { return &user @@ -718,7 +596,7 @@ func (kc *KubeConfig) findUser(name string) *KubeUser { return nil } -func (kc *KubeConfig) findCluster(name string) *KubeCluster { +func (kc *KubeConfig) FindCluster(name string) *KubeCluster { for _, cluster := range kc.Clusters { if cluster.Name == name { return &cluster @@ -727,55 +605,6 @@ func (kc *KubeConfig) findCluster(name string) *KubeCluster { return nil } -type E2EContext struct { - // Raw context name, - RawName string `yaml:"rawName"` - // A valid dns subdomain which can be used as the name of kubernetes resources. - Name string `yaml:"name"` - Cluster *KubeCluster `yaml:"cluster"` - User *KubeUser `yaml:"user"` -} - -func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { - if !f.federated { - Failf("getUnderlyingFederatedContexts called on non-federated framework") - } - - kubeconfig := KubeConfig{} - configBytes, err := ioutil.ReadFile(TestContext.KubeConfig) - ExpectNoError(err) - err = yaml.Unmarshal(configBytes, &kubeconfig) - ExpectNoError(err) - - e2eContexts := []E2EContext{} - for _, context := range kubeconfig.Contexts { - if strings.HasPrefix(context.Name, "federation") && context.Name != federatedKubeContext { - user := kubeconfig.findUser(context.Context.User) - if user == nil { - Failf("Could not find user for context %+v", context) - } - - cluster := kubeconfig.findCluster(context.Context.Cluster) - if cluster == nil { - Failf("Could not find cluster for context %+v", context) - } - - dnsSubdomainName, err := GetValidDNSSubdomainName(context.Name) - if err != nil { - Failf("Could not convert context name %s to a valid dns subdomain name, error: %s", context.Name, err) - } - e2eContexts = append(e2eContexts, E2EContext{ - RawName: context.Name, - Name: dnsSubdomainName, - Cluster: cluster, - User: user, - }) - } - } - - return e2eContexts -} - func kubectlExecWithRetry(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) { for numRetries := 0; numRetries < maxKubectlExecRetries; numRetries++ { if numRetries > 0 { diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 8b0b6dffcd7..bfdd7cc7add 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -81,6 +81,8 @@ type TestContextType struct { FeatureGates string // Node e2e specific test context NodeTestContextType + // Federation e2e context + FederatedKubeContext string // Viper-only parameters. These will in time replace all flags. @@ -129,7 +131,6 @@ type CloudConfig struct { } var TestContext TestContextType -var federatedKubeContext string // Register flags common to all e2e test suites. func RegisterCommonFlags() { @@ -163,7 +164,7 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver") - flag.StringVar(&federatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.") + flag.StringVar(&TestContext.FederatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.") flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index fc7c98b4dcf..1c4b7276b71 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -61,8 +61,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" - federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1" - "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1" @@ -361,23 +359,6 @@ func SkipUnlessServerVersionGTE(v *utilversion.Version, c discovery.ServerVersio } } -// Detects whether the federation namespace exists in the underlying cluster -func SkipUnlessFederated(c clientset.Interface) { - federationNS := os.Getenv("FEDERATION_NAMESPACE") - if federationNS == "" { - federationNS = federationapi.FederationNamespaceSystem - } - - _, err := c.Core().Namespaces().Get(federationNS, metav1.GetOptions{}) - if err != nil { - if apierrs.IsNotFound(err) { - Skipf("Could not find federation namespace %s: skipping federated test", federationNS) - } else { - Failf("Unexpected error getting namespace: %v", err) - } - } -} - func SkipIfMissingResource(clientPool dynamic.ClientPool, gvr schema.GroupVersionResource, namespace string) { dynamicClient, err := clientPool.ClientForGroupVersionResource(gvr) if err != nil { @@ -801,18 +782,6 @@ func WaitForDefaultServiceAccountInNamespace(c clientset.Interface, namespace st return waitForServiceAccountInNamespace(c, namespace, "default", ServiceAccountProvisionTimeout) } -// WaitForFederationApiserverReady waits for the federation apiserver to be ready. -// It tests the readiness by sending a GET request and expecting a non error response. -func WaitForFederationApiserverReady(c *federation_clientset.Clientset) error { - return wait.PollImmediate(time.Second, 1*time.Minute, func() (bool, error) { - _, err := c.Federation().Clusters().List(v1.ListOptions{}) - if err != nil { - return false, nil - } - return true, nil - }) -} - // WaitForPersistentVolumePhase waits for a PersistentVolume to be in a specific phase or until timeout occurs, whichever comes first. func WaitForPersistentVolumePhase(phase v1.PersistentVolumePhase, c clientset.Interface, pvName string, Poll, timeout time.Duration) error { Logf("Waiting up to %v for PersistentVolume %s to have phase %s", timeout, pvName, phase) @@ -1782,7 +1751,7 @@ func ServiceResponding(c clientset.Interface, ns, name string) error { }) } -func restclientConfig(kubeContext string) (*clientcmdapi.Config, error) { +func RestclientConfig(kubeContext string) (*clientcmdapi.Config, error) { Logf(">>> kubeConfig: %s\n", TestContext.KubeConfig) if TestContext.KubeConfig == "" { return nil, fmt.Errorf("KubeConfig must be specified to load client config") @@ -1805,43 +1774,13 @@ func LoadConfig() (*restclient.Config, error) { // This is a node e2e test, apply the node e2e configuration return &restclient.Config{Host: TestContext.Host}, nil } - c, err := restclientConfig(TestContext.KubeContext) + c, err := RestclientConfig(TestContext.KubeContext) if err != nil { return nil, err } return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: TestContext.Host}}).ClientConfig() } - -func LoadFederatedConfig(overrides *clientcmd.ConfigOverrides) (*restclient.Config, error) { - c, err := restclientConfig(federatedKubeContext) - if err != nil { - return nil, fmt.Errorf("error creating federation client config: %v", err.Error()) - } - cfg, err := clientcmd.NewDefaultClientConfig(*c, overrides).ClientConfig() - if cfg != nil { - //TODO(colhom): this is only here because https://github.com/kubernetes/kubernetes/issues/25422 - cfg.NegotiatedSerializer = api.Codecs - } - if err != nil { - return cfg, fmt.Errorf("error creating federation client config: %v", err.Error()) - } - return cfg, nil -} - -func LoadFederationClientset_1_5() (*federation_clientset.Clientset, error) { - config, err := LoadFederatedConfig(&clientcmd.ConfigOverrides{}) - if err != nil { - return nil, err - } - - c, err := federation_clientset.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("error creating federation clientset: %v", err.Error()) - } - return c, nil -} - func LoadInternalClientset() (*internalclientset.Clientset, error) { config, err := LoadConfig() if err != nil { diff --git a/test/e2e_federation/federated-daemonset.go b/test/e2e_federation/federated-daemonset.go index e48c0c36dc8..d7ace94792c 100644 --- a/test/e2e_federation/federated-daemonset.go +++ b/test/e2e_federation/federated-daemonset.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" ) const ( @@ -44,58 +45,58 @@ const ( var _ = framework.KubeDescribe("Federation daemonsets [Feature:Federation]", func() { var clusters map[string]*cluster // All clusters, keyed by cluster name - f := framework.NewDefaultFederatedFramework("federated-daemonset") + f := fedframework.NewDefaultFederatedFramework("federated-daemonset") Describe("DaemonSet objects", func() { BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) clusters = map[string]*cluster{} registerClusters(clusters, UserAgentName, "", f) }) AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Delete all daemonsets. nsName := f.FederationNamespace.Name - deleteAllDaemonSetsOrFail(f.FederationClientset_1_5, nsName) + deleteAllDaemonSetsOrFail(f.FederationClientset, nsName) unregisterClusters(clusters, f) }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - daemonset := createDaemonSetOrFail(f.FederationClientset_1_5, nsName) + daemonset := createDaemonSetOrFail(f.FederationClientset, nsName) defer func() { // Cleanup By(fmt.Sprintf("Deleting daemonset %q in namespace %q", daemonset.Name, nsName)) - err := f.FederationClientset_1_5.Extensions().DaemonSets(nsName).Delete(daemonset.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Extensions().DaemonSets(nsName).Delete(daemonset.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting daemonset %q in namespace %q", daemonset.Name, nsName) }() // wait for daemonset shards being created waitForDaemonSetShardsOrFail(nsName, daemonset, clusters) - daemonset = updateDaemonSetOrFail(f.FederationClientset_1_5, nsName) + daemonset = updateDaemonSetOrFail(f.FederationClientset, nsName) waitForDaemonSetShardsUpdatedOrFail(nsName, daemonset, clusters) }) It("should be deleted from underlying clusters when OrphanDependents is false", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := false - verifyCascadingDeletionForDS(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForDS(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that daemonsets were deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is true", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := true - verifyCascadingDeletionForDS(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForDS(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that daemonsets were not deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is nil", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - verifyCascadingDeletionForDS(f.FederationClientset_1_5, clusters, nil, nsName) + verifyCascadingDeletionForDS(f.FederationClientset, clusters, nil, nsName) By(fmt.Sprintf("Verified that daemonsets were not deleted from underlying clusters")) }) }) diff --git a/test/e2e_federation/federated-deployment.go b/test/e2e_federation/federated-deployment.go index ba40ccab167..2da2f041de1 100644 --- a/test/e2e_federation/federated-deployment.go +++ b/test/e2e_federation/federated-deployment.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -42,25 +43,25 @@ const ( // Create/delete deployment api objects var _ = framework.KubeDescribe("Federation deployments [Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federation-deployment") + f := fedframework.NewDefaultFederatedFramework("federation-deployment") Describe("Deployment objects", func() { AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Delete all deployments. nsName := f.FederationNamespace.Name - deleteAllDeploymentsOrFail(f.FederationClientset_1_5, nsName) + deleteAllDeploymentsOrFail(f.FederationClientset, nsName) }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - deployment := createDeploymentOrFail(f.FederationClientset_1_5, nsName) + deployment := createDeploymentOrFail(f.FederationClientset, nsName) By(fmt.Sprintf("Creation of deployment %q in namespace %q succeeded. Deleting deployment.", deployment.Name, nsName)) // Cleanup - err := f.FederationClientset_1_5.Extensions().Deployments(nsName).Delete(deployment.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Extensions().Deployments(nsName).Delete(deployment.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting deployment %q in namespace %q", deployment.Name, deployment.Namespace) By(fmt.Sprintf("Deletion of deployment %q in namespace %q succeeded.", deployment.Name, nsName)) }) @@ -74,7 +75,7 @@ var _ = framework.KubeDescribe("Federation deployments [Feature:Federation]", fu federationName string ) BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" { federationName = DefaultFederationName } @@ -84,50 +85,50 @@ var _ = framework.KubeDescribe("Federation deployments [Feature:Federation]", fu AfterEach(func() { nsName := f.FederationNamespace.Name - deleteAllDeploymentsOrFail(f.FederationClientset_1_5, nsName) + deleteAllDeploymentsOrFail(f.FederationClientset, nsName) unregisterClusters(clusters, f) }) It("should create and update matching deployments in underling clusters", func() { nsName := f.FederationNamespace.Name - dep := createDeploymentOrFail(f.FederationClientset_1_5, nsName) + dep := createDeploymentOrFail(f.FederationClientset, nsName) defer func() { // cleanup. deletion of deployments is not supported for underlying clusters By(fmt.Sprintf("Preparing deployment %q/%q for deletion by setting replicas to zero", nsName, dep.Name)) replicas := int32(0) dep.Spec.Replicas = &replicas - f.FederationClientset_1_5.Deployments(nsName).Update(dep) - waitForDeploymentOrFail(f.FederationClientset_1_5, nsName, dep.Name, clusters) - f.FederationClientset_1_5.Deployments(nsName).Delete(dep.Name, &v1.DeleteOptions{}) + f.FederationClientset.Deployments(nsName).Update(dep) + waitForDeploymentOrFail(f.FederationClientset, nsName, dep.Name, clusters) + f.FederationClientset.Deployments(nsName).Delete(dep.Name, &v1.DeleteOptions{}) }() - waitForDeploymentOrFail(f.FederationClientset_1_5, nsName, dep.Name, clusters) + waitForDeploymentOrFail(f.FederationClientset, nsName, dep.Name, clusters) By(fmt.Sprintf("Successfuly created and synced deployment %q/%q to clusters", nsName, dep.Name)) - updateDeploymentOrFail(f.FederationClientset_1_5, nsName) - waitForDeploymentOrFail(f.FederationClientset_1_5, nsName, dep.Name, clusters) + updateDeploymentOrFail(f.FederationClientset, nsName) + waitForDeploymentOrFail(f.FederationClientset, nsName, dep.Name, clusters) By(fmt.Sprintf("Successfuly updated and synced deployment %q/%q to clusters", nsName, dep.Name)) }) It("should be deleted from underlying clusters when OrphanDependents is false", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := false - verifyCascadingDeletionForDeployment(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForDeployment(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that deployments were deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is true", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := true - verifyCascadingDeletionForDeployment(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForDeployment(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that deployments were not deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is nil", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - verifyCascadingDeletionForDeployment(f.FederationClientset_1_5, clusters, nil, nsName) + verifyCascadingDeletionForDeployment(f.FederationClientset, clusters, nil, nsName) By(fmt.Sprintf("Verified that deployments were not deleted from underlying clusters")) }) diff --git a/test/e2e_federation/federated-ingress.go b/test/e2e_federation/federated-ingress.go index 6e9285188cc..171d8a05fd9 100644 --- a/test/e2e_federation/federated-ingress.go +++ b/test/e2e_federation/federated-ingress.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -34,6 +34,7 @@ import ( kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/util/intstr" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -47,8 +48,13 @@ const ( FederatedIngressServicePodName = "federated-ingress-service-test-pod" ) +const ( + // timeout on a single http request. + reqTimeout = 10 * time.Second +) + var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federated-ingress") + f := fedframework.NewDefaultFederatedFramework("federated-ingress") // Create/delete ingress api objects // Validate federation apiserver, does not rely on underlying clusters or federation ingress controller. @@ -56,17 +62,17 @@ var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func( AfterEach(func() { nsName := f.FederationNamespace.Name // Delete all ingresses. - deleteAllIngressesOrFail(f.FederationClientset_1_5, nsName) + deleteAllIngressesOrFail(f.FederationClientset, nsName) }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) framework.SkipUnlessProviderIs("gce", "gke") // TODO: Federated ingress is not yet supported on non-GCP platforms. nsName := f.FederationNamespace.Name - ingress := createIngressOrFail(f.FederationClientset_1_5, nsName) + ingress := createIngressOrFail(f.FederationClientset, nsName) By(fmt.Sprintf("Creation of ingress %q in namespace %q succeeded. Deleting ingress.", ingress.Name, nsName)) // Cleanup - err := f.FederationClientset_1_5.Extensions().Ingresses(nsName).Delete(ingress.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Extensions().Ingresses(nsName).Delete(ingress.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting ingress %q in namespace %q", ingress.Name, ingress.Namespace) By(fmt.Sprintf("Deletion of ingress %q in namespace %q succeeded.", ingress.Name, nsName)) }) @@ -82,12 +88,12 @@ var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func( // register clusters in federation apiserver BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) framework.SkipUnlessProviderIs("gce", "gke") // TODO: Federated ingress is not yet supported on non-GCP platforms. if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" { federationName = DefaultFederationName } - jig = newFederationTestJig(f.FederationClientset_1_5) + jig = newFederationTestJig(f.FederationClientset) clusters = map[string]*cluster{} primaryClusterName = registerClusters(clusters, UserAgentName, federationName, f) ns = f.FederationNamespace.Name @@ -96,38 +102,38 @@ var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func( AfterEach(func() { // Delete all ingresses. nsName := f.FederationNamespace.Name - deleteAllIngressesOrFail(f.FederationClientset_1_5, nsName) + deleteAllIngressesOrFail(f.FederationClientset, nsName) unregisterClusters(clusters, f) }) It("should create and update matching ingresses in underlying clusters", func() { - ingress := createIngressOrFail(f.FederationClientset_1_5, ns) + ingress := createIngressOrFail(f.FederationClientset, ns) // wait for ingress shards being created waitForIngressShardsOrFail(ns, ingress, clusters) - ingress = updateIngressOrFail(f.FederationClientset_1_5, ns) + ingress = updateIngressOrFail(f.FederationClientset, ns) waitForIngressShardsUpdatedOrFail(ns, ingress, clusters) }) It("should be deleted from underlying clusters when OrphanDependents is false", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := false - verifyCascadingDeletionForIngress(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForIngress(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that ingresses were deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is true", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := true - verifyCascadingDeletionForIngress(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForIngress(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that ingresses were not deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is nil", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - verifyCascadingDeletionForIngress(f.FederationClientset_1_5, clusters, nil, nsName) + verifyCascadingDeletionForIngress(f.FederationClientset, clusters, nil, nsName) By(fmt.Sprintf("Verified that ingresses were not deleted from underlying clusters")) }) @@ -138,13 +144,13 @@ var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func( ) BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // create backend pod createBackendPodsOrFail(clusters, ns, FederatedIngressServicePodName) // create backend service - service = createServiceOrFail(f.FederationClientset_1_5, ns, FederatedIngressServiceName) + service = createServiceOrFail(f.FederationClientset, ns, FederatedIngressServiceName) // create ingress object - jig.ing = createIngressOrFail(f.FederationClientset_1_5, ns) + jig.ing = createIngressOrFail(f.FederationClientset, ns) // wait for services objects sync waitForServiceShardsOrFail(ns, service, clusters) // wait for ingress objects sync @@ -154,14 +160,14 @@ var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func( AfterEach(func() { deleteBackendPodsOrFail(clusters, ns) if service != nil { - deleteServiceOrFail(f.FederationClientset_1_5, ns, service.Name, nil) + deleteServiceOrFail(f.FederationClientset, ns, service.Name, nil) cleanupServiceShardsAndProviderResources(ns, service, clusters) service = nil } else { By("No service to delete. Service is nil") } if jig.ing != nil { - deleteIngressOrFail(f.FederationClientset_1_5, ns, jig.ing.Name, nil) + deleteIngressOrFail(f.FederationClientset, ns, jig.ing.Name, nil) for clusterName, cluster := range clusters { deleteClusterIngressOrFail(clusterName, cluster.Clientset, ns, jig.ing.Name) } @@ -395,7 +401,7 @@ func updateIngressOrFail(clientset *fedclientset.Clientset, namespace string) (n for i := 0; i < MaxRetriesOnFederatedApiserver; i++ { newIng, err = clientset.Extensions().Ingresses(namespace).Update(ingress) if err == nil { - describeIng(namespace) + framework.DescribeIng(namespace) return newIng } if !errors.IsConflict(err) && !errors.IsServerTimeout(err) { @@ -422,7 +428,7 @@ func (j *federationTestJig) waitForFederatedIngress() { for _, p := range rules.IngressRuleValue.HTTP.Paths { route := fmt.Sprintf("%v://%v%v", proto, address, p.Path) framework.Logf("Testing route %v host %v with simple GET", route, rules.Host) - framework.ExpectNoError(pollURL(route, rules.Host, framework.LoadBalancerPollTimeout, framework.LoadBalancerPollInterval, timeoutClient, false)) + framework.ExpectNoError(framework.PollURL(route, rules.Host, framework.LoadBalancerPollTimeout, framework.LoadBalancerPollInterval, timeoutClient, false)) } } } diff --git a/test/e2e_federation/federated-namespace.go b/test/e2e_federation/federated-namespace.go index 2e5fc481396..b082456042f 100644 --- a/test/e2e_federation/federated-namespace.go +++ b/test/e2e_federation/federated-namespace.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" api_v1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" . "github.com/onsi/ginkgo" ) @@ -40,14 +41,14 @@ const ( // Create/delete ingress api objects var _ = framework.KubeDescribe("Federation namespace [Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federation-namespace") + f := fedframework.NewDefaultFederatedFramework("federation-namespace") Describe("Namespace objects", func() { var federationName string var clusters map[string]*cluster // All clusters, keyed by cluster name BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // TODO: Federation API server should be able to answer this. if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" { @@ -59,10 +60,10 @@ var _ = framework.KubeDescribe("Federation namespace [Feature:Federation]", func }) AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) deleteAllTestNamespaces(nil, - f.FederationClientset_1_5.Core().Namespaces().List, - f.FederationClientset_1_5.Core().Namespaces().Delete) + f.FederationClientset.Core().Namespaces().List, + f.FederationClientset.Core().Namespaces().Delete) for _, cluster := range clusters { deleteAllTestNamespaces(nil, cluster.Core().Namespaces().List, @@ -72,42 +73,42 @@ var _ = framework.KubeDescribe("Federation namespace [Feature:Federation]", func }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) - nsName := createNamespace(f.FederationClientset_1_5.Core().Namespaces()) + nsName := createNamespace(f.FederationClientset.Core().Namespaces()) By(fmt.Sprintf("Deleting namespace %s", nsName)) deleteAllTestNamespaces(nil, - f.FederationClientset_1_5.Core().Namespaces().List, - f.FederationClientset_1_5.Core().Namespaces().Delete) + f.FederationClientset.Core().Namespaces().List, + f.FederationClientset.Core().Namespaces().Delete) By(fmt.Sprintf("Verified that deletion succeeded")) }) It("should be deleted from underlying clusters when OrphanDependents is false", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) orphanDependents := false - verifyNsCascadingDeletion(f.FederationClientset_1_5.Core().Namespaces(), clusters, &orphanDependents) + verifyNsCascadingDeletion(f.FederationClientset.Core().Namespaces(), clusters, &orphanDependents) By(fmt.Sprintf("Verified that namespaces were deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is true", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) orphanDependents := true - verifyNsCascadingDeletion(f.FederationClientset_1_5.Core().Namespaces(), clusters, &orphanDependents) + verifyNsCascadingDeletion(f.FederationClientset.Core().Namespaces(), clusters, &orphanDependents) By(fmt.Sprintf("Verified that namespaces were not deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is nil", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) - verifyNsCascadingDeletion(f.FederationClientset_1_5.Core().Namespaces(), clusters, nil) + verifyNsCascadingDeletion(f.FederationClientset.Core().Namespaces(), clusters, nil) By(fmt.Sprintf("Verified that namespaces were not deleted from underlying clusters")) }) It("all resources in the namespace should be deleted when namespace is deleted", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) - nsName := createNamespace(f.FederationClientset_1_5.Core().Namespaces()) + nsName := createNamespace(f.FederationClientset.Core().Namespaces()) // Create resources in the namespace. event := api_v1.Event{ @@ -122,18 +123,18 @@ var _ = framework.KubeDescribe("Federation namespace [Feature:Federation]", func }, } By(fmt.Sprintf("Creating event %s in namespace %s", event.Name, nsName)) - _, err := f.FederationClientset_1_5.Core().Events(nsName).Create(&event) + _, err := f.FederationClientset.Core().Events(nsName).Create(&event) if err != nil { framework.Failf("Failed to create event %v in namespace %s, err: %s", event, nsName, err) } By(fmt.Sprintf("Deleting namespace %s", nsName)) deleteAllTestNamespaces(nil, - f.FederationClientset_1_5.Core().Namespaces().List, - f.FederationClientset_1_5.Core().Namespaces().Delete) + f.FederationClientset.Core().Namespaces().List, + f.FederationClientset.Core().Namespaces().Delete) By(fmt.Sprintf("Verify that event %s was deleted as well", event.Name)) - latestEvent, err := f.FederationClientset_1_5.Core().Events(nsName).Get(event.Name, metav1.GetOptions{}) + latestEvent, err := f.FederationClientset.Core().Events(nsName).Get(event.Name, metav1.GetOptions{}) if !errors.IsNotFound(err) { framework.Failf("Event %s should have been deleted. Found: %v", event.Name, latestEvent) } diff --git a/test/e2e_federation/federated-replicaset.go b/test/e2e_federation/federated-replicaset.go index 2bf24f6ab85..4544ab7221e 100644 --- a/test/e2e_federation/federated-replicaset.go +++ b/test/e2e_federation/federated-replicaset.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" "reflect" @@ -44,25 +45,25 @@ const ( // Create/delete replicaset api objects var _ = framework.KubeDescribe("Federation replicasets [Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federation-replicaset") + f := fedframework.NewDefaultFederatedFramework("federation-replicaset") Describe("ReplicaSet objects", func() { AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Delete all replicasets. nsName := f.FederationNamespace.Name - deleteAllReplicaSetsOrFail(f.FederationClientset_1_5, nsName) + deleteAllReplicaSetsOrFail(f.FederationClientset, nsName) }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - replicaset := createReplicaSetOrFail(f.FederationClientset_1_5, nsName) + replicaset := createReplicaSetOrFail(f.FederationClientset, nsName) By(fmt.Sprintf("Creation of replicaset %q in namespace %q succeeded. Deleting replicaset.", replicaset.Name, nsName)) // Cleanup - err := f.FederationClientset_1_5.Extensions().ReplicaSets(nsName).Delete(replicaset.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Extensions().ReplicaSets(nsName).Delete(replicaset.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting replicaset %q in namespace %q", replicaset.Name, replicaset.Namespace) By(fmt.Sprintf("Deletion of replicaset %q in namespace %q succeeded.", replicaset.Name, nsName)) }) @@ -76,7 +77,7 @@ var _ = framework.KubeDescribe("Federation replicasets [Feature:Federation]", fu federationName string ) BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" { federationName = DefaultFederationName } @@ -87,50 +88,50 @@ var _ = framework.KubeDescribe("Federation replicasets [Feature:Federation]", fu AfterEach(func() { // Delete all replicasets. nsName := f.FederationNamespace.Name - deleteAllReplicaSetsOrFail(f.FederationClientset_1_5, nsName) + deleteAllReplicaSetsOrFail(f.FederationClientset, nsName) unregisterClusters(clusters, f) }) It("should create and update matching replicasets in underling clusters", func() { nsName := f.FederationNamespace.Name - rs := createReplicaSetOrFail(f.FederationClientset_1_5, nsName) + rs := createReplicaSetOrFail(f.FederationClientset, nsName) defer func() { // cleanup. deletion of replicasets is not supported for underlying clusters By(fmt.Sprintf("Preparing replicaset %q/%q for deletion by setting replicas to zero", nsName, rs.Name)) replicas := int32(0) rs.Spec.Replicas = &replicas - f.FederationClientset_1_5.ReplicaSets(nsName).Update(rs) - waitForReplicaSetOrFail(f.FederationClientset_1_5, nsName, rs.Name, clusters) - f.FederationClientset_1_5.ReplicaSets(nsName).Delete(rs.Name, &v1.DeleteOptions{}) + f.FederationClientset.ReplicaSets(nsName).Update(rs) + waitForReplicaSetOrFail(f.FederationClientset, nsName, rs.Name, clusters) + f.FederationClientset.ReplicaSets(nsName).Delete(rs.Name, &v1.DeleteOptions{}) }() - waitForReplicaSetOrFail(f.FederationClientset_1_5, nsName, rs.Name, clusters) + waitForReplicaSetOrFail(f.FederationClientset, nsName, rs.Name, clusters) By(fmt.Sprintf("Successfuly created and synced replicaset %q/%q to clusters", nsName, rs.Name)) - updateReplicaSetOrFail(f.FederationClientset_1_5, nsName) - waitForReplicaSetOrFail(f.FederationClientset_1_5, nsName, rs.Name, clusters) + updateReplicaSetOrFail(f.FederationClientset, nsName) + waitForReplicaSetOrFail(f.FederationClientset, nsName, rs.Name, clusters) By(fmt.Sprintf("Successfuly updated and synced replicaset %q/%q to clusters", nsName, rs.Name)) }) It("should be deleted from underlying clusters when OrphanDependents is false", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := false - verifyCascadingDeletionForReplicaSet(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForReplicaSet(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that replica sets were deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is true", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := true - verifyCascadingDeletionForReplicaSet(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForReplicaSet(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that replica sets were not deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is nil", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - verifyCascadingDeletionForReplicaSet(f.FederationClientset_1_5, clusters, nil, nsName) + verifyCascadingDeletionForReplicaSet(f.FederationClientset, clusters, nil, nsName) By(fmt.Sprintf("Verified that replica sets were not deleted from underlying clusters")) }) }) diff --git a/test/e2e_federation/federated-secret.go b/test/e2e_federation/federated-secret.go index 5d17139e20a..548e62a2618 100644 --- a/test/e2e_federation/federated-secret.go +++ b/test/e2e_federation/federated-secret.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -31,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" ) const ( @@ -43,55 +44,55 @@ const ( var _ = framework.KubeDescribe("Federation secrets [Feature:Federation]", func() { var clusters map[string]*cluster // All clusters, keyed by cluster name - f := framework.NewDefaultFederatedFramework("federated-secret") + f := fedframework.NewDefaultFederatedFramework("federated-secret") Describe("Secret objects", func() { BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) clusters = map[string]*cluster{} registerClusters(clusters, UserAgentName, "", f) }) AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Delete all secrets. nsName := f.FederationNamespace.Name - deleteAllSecretsOrFail(f.FederationClientset_1_5, nsName) + deleteAllSecretsOrFail(f.FederationClientset, nsName) unregisterClusters(clusters, f) }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - secret := createSecretOrFail(f.FederationClientset_1_5, nsName) + secret := createSecretOrFail(f.FederationClientset, nsName) // wait for secret shards being created waitForSecretShardsOrFail(nsName, secret, clusters) - secret = updateSecretOrFail(f.FederationClientset_1_5, nsName, secret.Name) + secret = updateSecretOrFail(f.FederationClientset, nsName, secret.Name) waitForSecretShardsUpdatedOrFail(nsName, secret, clusters) }) It("should be deleted from underlying clusters when OrphanDependents is false", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := false - verifyCascadingDeletionForSecret(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForSecret(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that secrets were deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is true", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := true - verifyCascadingDeletionForSecret(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForSecret(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that secrets were not deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is nil", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - verifyCascadingDeletionForSecret(f.FederationClientset_1_5, clusters, nil, nsName) + verifyCascadingDeletionForSecret(f.FederationClientset, clusters, nil, nsName) By(fmt.Sprintf("Verified that secrets were not deleted from underlying clusters")) }) }) diff --git a/test/e2e_federation/federated-service.go b/test/e2e_federation/federated-service.go index 18b38f94fb4..919726ec752 100644 --- a/test/e2e_federation/federated-service.go +++ b/test/e2e_federation/federated-service.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -31,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" . "github.com/onsi/ginkgo" ) @@ -50,14 +51,14 @@ var FederatedServiceLabels = map[string]string{ } var _ = framework.KubeDescribe("[Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federated-service") + f := fedframework.NewDefaultFederatedFramework("federated-service") var clusters map[string]*cluster // All clusters, keyed by cluster name var federationName string var primaryClusterName string // The name of the "primary" cluster var _ = Describe("Federated Services", func() { BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // TODO: Federation API server should be able to answer this. if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" { @@ -79,12 +80,12 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { ) BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Placeholder }) AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) if service != nil { By(fmt.Sprintf("Deleting service shards and their provider resources in underlying clusters for service %q in namespace %q", service.Name, nsName)) @@ -95,51 +96,51 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { }) It("should succeed", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName = f.FederationNamespace.Name - service = createServiceOrFail(f.FederationClientset_1_5, nsName, FederatedServiceName) + service = createServiceOrFail(f.FederationClientset, nsName, FederatedServiceName) By(fmt.Sprintf("Creation of service %q in namespace %q succeeded. Deleting service.", service.Name, nsName)) // Cleanup - err := f.FederationClientset_1_5.Services(nsName).Delete(service.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Services(nsName).Delete(service.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting service %q in namespace %q", service.Name, service.Namespace) By(fmt.Sprintf("Deletion of service %q in namespace %q succeeded.", service.Name, nsName)) }) It("should create matching services in underlying clusters", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName = f.FederationNamespace.Name - service = createServiceOrFail(f.FederationClientset_1_5, nsName, FederatedServiceName) + service = createServiceOrFail(f.FederationClientset, nsName, FederatedServiceName) defer func() { // Cleanup By(fmt.Sprintf("Deleting service %q in namespace %q", service.Name, nsName)) - err := f.FederationClientset_1_5.Services(nsName).Delete(service.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Services(nsName).Delete(service.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting service %q in namespace %q", service.Name, nsName) }() waitForServiceShardsOrFail(nsName, service, clusters) }) It("should be deleted from underlying clusters when OrphanDependents is false", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := false - verifyCascadingDeletionForService(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForService(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that services were deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is true", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name orphanDependents := true - verifyCascadingDeletionForService(f.FederationClientset_1_5, clusters, &orphanDependents, nsName) + verifyCascadingDeletionForService(f.FederationClientset, clusters, &orphanDependents, nsName) By(fmt.Sprintf("Verified that services were not deleted from underlying clusters")) }) It("should not be deleted from underlying clusters when OrphanDependents is nil", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - verifyCascadingDeletionForService(f.FederationClientset_1_5, clusters, nil, nsName) + verifyCascadingDeletionForService(f.FederationClientset, clusters, nil, nsName) By(fmt.Sprintf("Verified that services were not deleted from underlying clusters")) }) }) @@ -151,7 +152,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { ) BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name // Create kube-dns configmap for kube-dns to accept federation queries. @@ -177,7 +178,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { createBackendPodsOrFail(clusters, nsName, FederatedServicePodName) - service = createServiceOrFail(f.FederationClientset_1_5, nsName, FederatedServiceName) + service = createServiceOrFail(f.FederationClientset, nsName, FederatedServiceName) obj, err := api.Scheme.DeepCopy(service) // Cloning shouldn't fail. On the off-chance it does, we // should shallow copy service to serviceShard before @@ -210,13 +211,13 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { }) AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name deleteBackendPodsOrFail(clusters, nsName) if service != nil { - deleteServiceOrFail(f.FederationClientset_1_5, nsName, service.Name, nil) + deleteServiceOrFail(f.FederationClientset, nsName, service.Name, nil) service = nil } else { By("No service to delete. Service is nil") @@ -239,7 +240,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { }) It("should be able to discover a federated service", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name svcDNSNames := []string{ @@ -258,7 +259,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { By("Verified that DNS rules are working as expected") By("Deleting the service to verify that DNS rules still work") - err := f.FederationClientset_1_5.Services(nsName).Delete(FederatedServiceName, &v1.DeleteOptions{}) + err := f.FederationClientset.Services(nsName).Delete(FederatedServiceName, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting service %q in namespace %q", service.Name, service.Namespace) // Service is deleted, unset the test block-global service variable. service = nil @@ -271,7 +272,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { Context("non-local federated service", func() { BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Delete all the backend pods from the shard which is local to the discovery pod. deleteOneBackendPodOrFail(clusters[primaryClusterName]) @@ -279,7 +280,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { }) It("should be able to discover a non-local federated service", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name svcDNSNames := []string{ @@ -295,7 +296,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { // TTL and/or running the pods in parallel. Context("[Slow] missing local service", func() { It("should never find DNS entries for a missing local service", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name localSvcDNSNames := []string{ diff --git a/test/e2e_federation/federation-apiserver.go b/test/e2e_federation/federation-apiserver.go index ae972f58563..9b0702e48a1 100644 --- a/test/e2e_federation/federation-apiserver.go +++ b/test/e2e_federation/federation-apiserver.go @@ -14,43 +14,43 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" "time" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1" "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/test/e2e/framework" - - . "github.com/onsi/gomega" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" ) // Create/delete cluster api objects var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federation-cluster") + f := fedframework.NewDefaultFederatedFramework("federation-cluster") Describe("Cluster objects", func() { AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Delete registered clusters. // This is if a test failed, it should not affect other tests. - clusterList, err := f.FederationClientset_1_5.Federation().Clusters().List(v1.ListOptions{}) + clusterList, err := f.FederationClientset.Federation().Clusters().List(v1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) for _, cluster := range clusterList.Items { - err := f.FederationClientset_1_5.Federation().Clusters().Delete(cluster.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Federation().Clusters().Delete(cluster.Name, &v1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) } }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) contexts := f.GetUnderlyingFederatedContexts() @@ -69,14 +69,14 @@ var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func framework.Logf("Deleting %d clusters", len(contexts)) for _, context := range contexts { framework.Logf("Deleting cluster object: %s (%s, secret: %s)", context.Name, context.Cluster.Cluster.Server, context.Name) - err := f.FederationClientset_1_5.Federation().Clusters().Delete(context.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Federation().Clusters().Delete(context.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, fmt.Sprintf("unexpected error in deleting cluster %s: %+v", context.Name, err)) framework.Logf("Successfully deleted cluster object: %s (%s, secret: %s)", context.Name, context.Cluster.Cluster.Server, context.Name) } // There should not be any remaining cluster. framework.Logf("Verifying that zero clusters remain") - clusterList, err := f.FederationClientset_1_5.Federation().Clusters().List(v1.ListOptions{}) + clusterList, err := f.FederationClientset.Federation().Clusters().List(v1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) if len(clusterList.Items) != 0 { framework.Failf("there should not have been any remaining clusters. Found: %+v", clusterList) @@ -86,16 +86,16 @@ var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func }) Describe("Admission control", func() { AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) }) It("should not be able to create resources if namespace does not exist", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) // Creating a service in a non-existing namespace should fail. svcNamespace := "federation-admission-test-ns" svcName := "myns" - clientset := f.FederationClientset_1_5 + clientset := f.FederationClientset framework.Logf("Trying to create service %s in namespace %s, expect to get error", svcName, svcNamespace) if _, err := clientset.Core().Services(svcNamespace).Create(newService(svcName, svcNamespace)); err == nil { framework.Failf("Expected to get an error while creating a service in a non-existing namespace") diff --git a/test/e2e_federation/federation-authn.go b/test/e2e_federation/federation-authn.go index 1f0e8e86da9..f1096715db8 100644 --- a/test/e2e_federation/federation-authn.go +++ b/test/e2e_federation/federation-authn.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -24,29 +24,30 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = framework.KubeDescribe("[Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federation-apiserver-authn") + f := fedframework.NewDefaultFederatedFramework("federation-apiserver-authn") var _ = Describe("Federation API server authentication", func() { BeforeEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) }) It("should accept cluster resources when the client has right authentication credentials", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - svc := createServiceOrFail(f.FederationClientset_1_5, nsName, FederatedServiceName) - deleteServiceOrFail(f.FederationClientset_1_5, nsName, svc.Name, nil) + svc := createServiceOrFail(f.FederationClientset, nsName, FederatedServiceName) + deleteServiceOrFail(f.FederationClientset, nsName, svc.Name, nil) }) It("should not accept cluster resources when the client has invalid authentication credentials", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) contexts := f.GetUnderlyingFederatedContexts() @@ -67,7 +68,7 @@ var _ = framework.KubeDescribe("[Feature:Federation]", func() { }) It("should not accept cluster resources when the client has no authentication credentials", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) fcs, err := invalidAuthFederationClientSet(nil) framework.ExpectNoError(err) @@ -94,7 +95,7 @@ func invalidAuthFederationClientSet(user *framework.KubeUser) (*federation_clien } } - config, err := framework.LoadFederatedConfig(overrides) + config, err := fedframework.LoadFederatedConfig(overrides) if err != nil { return nil, err } diff --git a/test/e2e_federation/federation-event.go b/test/e2e_federation/federation-event.go index 7c1c2af8260..9a7ada6450e 100644 --- a/test/e2e_federation/federation-event.go +++ b/test/e2e_federation/federation-event.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -24,6 +24,7 @@ import ( "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" . "github.com/onsi/gomega" ) @@ -34,30 +35,30 @@ const ( // Create/delete event api objects. var _ = framework.KubeDescribe("Federation events [Feature:Federation]", func() { - f := framework.NewDefaultFederatedFramework("federation-event") + f := fedframework.NewDefaultFederatedFramework("federation-event") Describe("Event objects", func() { AfterEach(func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name // Delete registered events. - eventList, err := f.FederationClientset_1_5.Core().Events(nsName).List(v1.ListOptions{}) + eventList, err := f.FederationClientset.Core().Events(nsName).List(v1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) for _, event := range eventList.Items { - err := f.FederationClientset_1_5.Core().Events(nsName).Delete(event.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Core().Events(nsName).Delete(event.Name, &v1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) } }) It("should be created and deleted successfully", func() { - framework.SkipUnlessFederated(f.ClientSet) + fedframework.SkipUnlessFederated(f.ClientSet) nsName := f.FederationNamespace.Name - event := createEventOrFail(f.FederationClientset_1_5, nsName) + event := createEventOrFail(f.FederationClientset, nsName) By(fmt.Sprintf("Creation of event %q in namespace %q succeeded. Deleting event.", event.Name, nsName)) // Cleanup - err := f.FederationClientset_1_5.Core().Events(nsName).Delete(event.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Core().Events(nsName).Delete(event.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting event %q in namespace %q", event.Name, event.Namespace) By(fmt.Sprintf("Deletion of event %q in namespace %q succeeded.", event.Name, nsName)) }) diff --git a/test/e2e_federation/federation-util.go b/test/e2e_federation/federation-util.go index 8aeb04b02ba..7f164bbb0fc 100644 --- a/test/e2e_federation/federation-util.go +++ b/test/e2e_federation/federation-util.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_federation import ( "fmt" @@ -33,7 +33,9 @@ import ( "k8s.io/kubernetes/pkg/client/restclient" "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -48,6 +50,8 @@ var ( DNSTTL = 180 * time.Second // TODO: make k8s.io/kubernetes/federation/pkg/federation-controller/service.minDnsTtl exported, and import it here. ) +var FederationSuite common.Suite + /* cluster keeps track of the assorted objects and state related to each cluster in the federation @@ -59,7 +63,7 @@ type cluster struct { backendPod *v1.Pod // The backend pod, if one's been created. } -func createClusterObjectOrFail(f *framework.Framework, context *framework.E2EContext) { +func createClusterObjectOrFail(f *fedframework.Framework, context *fedframework.E2EContext) { framework.Logf("Creating cluster object: %s (%s, secret: %s)", context.Name, context.Cluster.Cluster.Server, context.Name) cluster := federationapi.Cluster{ ObjectMeta: metav1.ObjectMeta{ @@ -80,29 +84,29 @@ func createClusterObjectOrFail(f *framework.Framework, context *framework.E2ECon }, }, } - _, err := f.FederationClientset_1_5.Federation().Clusters().Create(&cluster) + _, err := f.FederationClientset.Federation().Clusters().Create(&cluster) framework.ExpectNoError(err, fmt.Sprintf("creating cluster: %+v", err)) framework.Logf("Successfully created cluster object: %s (%s, secret: %s)", context.Name, context.Cluster.Cluster.Server, context.Name) } -func clusterIsReadyOrFail(f *framework.Framework, context *framework.E2EContext) { - c, err := f.FederationClientset_1_5.Federation().Clusters().Get(context.Name, metav1.GetOptions{}) +func clusterIsReadyOrFail(f *fedframework.Framework, context *fedframework.E2EContext) { + c, err := f.FederationClientset.Federation().Clusters().Get(context.Name, metav1.GetOptions{}) framework.ExpectNoError(err, fmt.Sprintf("get cluster: %+v", err)) if c.ObjectMeta.Name != context.Name { framework.Failf("cluster name does not match input context: actual=%+v, expected=%+v", c, context) } - err = isReady(context.Name, f.FederationClientset_1_5) + err = isReady(context.Name, f.FederationClientset) framework.ExpectNoError(err, fmt.Sprintf("unexpected error in verifying if cluster %s is ready: %+v", context.Name, err)) framework.Logf("Cluster %s is Ready", context.Name) } // waitForAllClustersReady wait for all clusters defined in e2e context to be created // return ClusterList until the listed cluster items equals clusterCount -func waitForAllClustersReady(f *framework.Framework, clusterCount int) *federationapi.ClusterList { +func waitForAllClustersReady(f *fedframework.Framework, clusterCount int) *federationapi.ClusterList { var clusterList *federationapi.ClusterList if err := wait.PollImmediate(framework.Poll, FederatedServiceTimeout, func() (bool, error) { var err error - clusterList, err = f.FederationClientset_1_5.Federation().Clusters().List(v1.ListOptions{}) + clusterList, err = f.FederationClientset.Federation().Clusters().List(v1.ListOptions{}) if err != nil { return false, err } @@ -136,7 +140,7 @@ func createClientsetForCluster(c federationapi.Cluster, i int, userAgentName str } // Creates the federation namespace in all underlying clusters. -func createNamespaceInClusters(clusters map[string]*cluster, f *framework.Framework) { +func createNamespaceInClusters(clusters map[string]*cluster, f *fedframework.Framework) { nsName := f.FederationNamespace.Name for name, c := range clusters { // The e2e Framework created the required namespace in federation control plane, but we need to create it in all the others, if it doesn't yet exist. @@ -161,7 +165,7 @@ func createNamespaceInClusters(clusters map[string]*cluster, f *framework.Framew // Unregisters the given clusters from federation control plane. // Also deletes the federation namespace from each cluster. -func unregisterClusters(clusters map[string]*cluster, f *framework.Framework) { +func unregisterClusters(clusters map[string]*cluster, f *fedframework.Framework) { nsName := f.FederationNamespace.Name for name, c := range clusters { if c.namespaceCreated { @@ -174,16 +178,16 @@ func unregisterClusters(clusters map[string]*cluster, f *framework.Framework) { } // Delete the registered clusters in the federation API server. - clusterList, err := f.FederationClientset_1_5.Federation().Clusters().List(v1.ListOptions{}) + clusterList, err := f.FederationClientset.Federation().Clusters().List(v1.ListOptions{}) framework.ExpectNoError(err, "Error listing clusters") for _, cluster := range clusterList.Items { - err := f.FederationClientset_1_5.Federation().Clusters().Delete(cluster.Name, &v1.DeleteOptions{}) + err := f.FederationClientset.Federation().Clusters().Delete(cluster.Name, &v1.DeleteOptions{}) framework.ExpectNoError(err, "Error deleting cluster %q", cluster.Name) } } // can not be moved to util, as By and Expect must be put in Ginkgo test unit -func registerClusters(clusters map[string]*cluster, userAgentName, federationName string, f *framework.Framework) string { +func registerClusters(clusters map[string]*cluster, userAgentName, federationName string, f *fedframework.Framework) string { contexts := f.GetUnderlyingFederatedContexts() for _, context := range contexts { @@ -382,7 +386,7 @@ func cleanupServiceShardLoadBalancer(clusterName string, service *v1.Service, ti return err } -func podExitCodeDetector(f *framework.Framework, name, namespace string, code int32) func() error { +func podExitCodeDetector(f *fedframework.Framework, name, namespace string, code int32) func() error { // If we ever get any container logs, stash them here. logs := "" @@ -423,7 +427,7 @@ func podExitCodeDetector(f *framework.Framework, name, namespace string, code in } } -func discoverService(f *framework.Framework, name string, exists bool, podName string) { +func discoverService(f *fedframework.Framework, name string, exists bool, podName string) { command := []string{"sh", "-c", fmt.Sprintf("until nslookup '%s'; do sleep 10; done", name)} By(fmt.Sprintf("Looking up %q", name)) diff --git a/test/e2e_federation/framework/framework.go b/test/e2e_federation/framework/framework.go new file mode 100644 index 00000000000..bf1633bca88 --- /dev/null +++ b/test/e2e_federation/framework/framework.go @@ -0,0 +1,213 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "io/ioutil" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" + "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + yaml "gopkg.in/yaml.v2" +) + +// Framework extends e2e Framework and adds federation specific fields +type Framework struct { + *framework.Framework + + FederationClientset *federation_clientset.Clientset + FederationNamespace *v1.Namespace +} + +func NewDefaultFederatedFramework(baseName string) *Framework { + f := &Framework{framework.NewDefaultFramework(baseName), nil, &v1.Namespace{}} + + BeforeEach(f.FederationBeforeEach) + AfterEach(f.FederationAfterEach) + + return f +} + +// FederationBeforeEach checks for federation apiserver is ready and makes a namespace. +func (f *Framework) FederationBeforeEach() { + if f.FederationClientset == nil { + By("Creating a release 1.5 federation Clientset") + var err error + f.FederationClientset, err = LoadFederationClientset() + Expect(err).NotTo(HaveOccurred()) + } + By("Waiting for federation-apiserver to be ready") + err := WaitForFederationApiserverReady(f.FederationClientset) + Expect(err).NotTo(HaveOccurred()) + By("federation-apiserver is ready") + + By("Creating a federation namespace") + ns, err := f.createFederationNamespace(f.BaseName) + Expect(err).NotTo(HaveOccurred()) + f.FederationNamespace = ns + By(fmt.Sprintf("Created federation namespace %s", ns.Name)) +} + +func (f *Framework) deleteFederationNs() { + ns := f.FederationNamespace + By(fmt.Sprintf("Destroying federation namespace %q for this suite.", ns.Name)) + timeout := 5 * time.Minute + if f.NamespaceDeletionTimeout != 0 { + timeout = f.NamespaceDeletionTimeout + } + + clientset := f.FederationClientset + // First delete the namespace from federation apiserver. + // Also delete the corresponding namespaces from underlying clusters. + orphanDependents := false + if err := clientset.Core().Namespaces().Delete(ns.Name, &v1.DeleteOptions{OrphanDependents: &orphanDependents}); err != nil { + framework.Failf("Error while deleting federation namespace %s: %s", ns.Name, err) + } + // Verify that it got deleted. + err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { + if _, err := clientset.Core().Namespaces().Get(ns.Name, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + framework.Logf("Error while waiting for namespace to be terminated: %v", err) + return false, nil + } + return false, nil + }) + if err != nil { + if !apierrors.IsNotFound(err) { + framework.Failf("Couldn't delete ns %q: %s", ns.Name, err) + } else { + framework.Logf("Namespace %v was already deleted", ns.Name) + } + } +} + +// FederationAfterEach deletes the namespace, after reading its events. +func (f *Framework) FederationAfterEach() { + // DeleteNamespace at the very end in defer, to avoid any + // expectation failures preventing deleting the namespace. + defer func() { + // Whether to delete namespace is determined by 3 factors: delete-namespace flag, delete-namespace-on-failure flag and the test result + // if delete-namespace set to false, namespace will always be preserved. + // if delete-namespace is true and delete-namespace-on-failure is false, namespace will be preserved if test failed. + if framework.TestContext.DeleteNamespace && (framework.TestContext.DeleteNamespaceOnFailure || !CurrentGinkgoTestDescription().Failed) { + // Delete the federation namespace. + f.deleteFederationNs() + } + + // Paranoia-- prevent reuse! + f.FederationNamespace = nil + + if f.FederationClientset == nil { + framework.Logf("Warning: framework is marked federated, but has no federation 1.5 clientset") + return + } + if err := f.FederationClientset.Federation().Clusters().DeleteCollection(nil, v1.ListOptions{}); err != nil { + framework.Logf("Error: failed to delete Clusters: %+v", err) + } + }() + + // Print events if the test failed. + if CurrentGinkgoTestDescription().Failed && framework.TestContext.DumpLogsOnFailure { + // Dump federation events in federation namespace. + framework.DumpEventsInNamespace(func(opts v1.ListOptions, ns string) (*v1.EventList, error) { + return f.FederationClientset.Core().Events(ns).List(opts) + }, f.FederationNamespace.Name) + // Print logs of federation control plane pods (federation-apiserver and federation-controller-manager) + framework.LogPodsWithLabels(f.ClientSet, "federation", map[string]string{"app": "federated-cluster"}, framework.Logf) + // Print logs of kube-dns pod + framework.LogPodsWithLabels(f.ClientSet, "kube-system", map[string]string{"k8s-app": "kube-dns"}, framework.Logf) + } +} + +func (f *Framework) createFederationNamespace(baseName string) (*v1.Namespace, error) { + clientset := f.FederationClientset + namespaceObj := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("e2e-tests-%v-", baseName), + }, + } + // Be robust about making the namespace creation call. + var got *v1.Namespace + if err := wait.PollImmediate(framework.Poll, framework.SingleCallTimeout, func() (bool, error) { + var err error + got, err = clientset.Core().Namespaces().Create(namespaceObj) + if err != nil { + framework.Logf("Unexpected error while creating namespace: %v", err) + return false, nil + } + return true, nil + }); err != nil { + return nil, err + } + return got, nil +} + +type E2EContext struct { + // Raw context name, + RawName string `yaml:"rawName"` + // A valid dns subdomain which can be used as the name of kubernetes resources. + Name string `yaml:"name"` + Cluster *framework.KubeCluster `yaml:"cluster"` + User *framework.KubeUser `yaml:"user"` +} + +func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { + kubeconfig := framework.KubeConfig{} + configBytes, err := ioutil.ReadFile(framework.TestContext.KubeConfig) + framework.ExpectNoError(err) + err = yaml.Unmarshal(configBytes, &kubeconfig) + framework.ExpectNoError(err) + + e2eContexts := []E2EContext{} + for _, context := range kubeconfig.Contexts { + if strings.HasPrefix(context.Name, "federation") && context.Name != framework.TestContext.FederatedKubeContext { + user := kubeconfig.FindUser(context.Context.User) + if user == nil { + framework.Failf("Could not find user for context %+v", context) + } + + cluster := kubeconfig.FindCluster(context.Context.Cluster) + if cluster == nil { + framework.Failf("Could not find cluster for context %+v", context) + } + + dnsSubdomainName, err := GetValidDNSSubdomainName(context.Name) + if err != nil { + framework.Failf("Could not convert context name %s to a valid dns subdomain name, error: %s", context.Name, err) + } + e2eContexts = append(e2eContexts, E2EContext{ + RawName: context.Name, + Name: dnsSubdomainName, + Cluster: cluster, + User: user, + }) + } + } + + return e2eContexts +} diff --git a/test/e2e_federation/framework/util.go b/test/e2e_federation/framework/util.go new file mode 100644 index 00000000000..5dd8d850743 --- /dev/null +++ b/test/e2e_federation/framework/util.go @@ -0,0 +1,113 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "os" + "regexp" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + validationutil "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/wait" + federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1" + "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/pkg/api/validation" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + "k8s.io/kubernetes/test/e2e/framework" +) + +// Detects whether the federation namespace exists in the underlying cluster +func SkipUnlessFederated(c clientset.Interface) { + federationNS := os.Getenv("FEDERATION_NAMESPACE") + if federationNS == "" { + federationNS = federationapi.FederationNamespaceSystem + } + + _, err := c.Core().Namespaces().Get(federationNS, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + framework.Skipf("Could not find federation namespace %s: skipping federated test", federationNS) + } else { + framework.Failf("Unexpected error getting namespace: %v", err) + } + } +} + +// WaitForFederationApiserverReady waits for the federation apiserver to be ready. +// It tests the readiness by sending a GET request and expecting a non error response. +func WaitForFederationApiserverReady(c *federation_clientset.Clientset) error { + return wait.PollImmediate(time.Second, 1*time.Minute, func() (bool, error) { + _, err := c.Federation().Clusters().List(v1.ListOptions{}) + if err != nil { + return false, nil + } + return true, nil + }) +} + +func LoadFederationClientset() (*federation_clientset.Clientset, error) { + config, err := LoadFederatedConfig(&clientcmd.ConfigOverrides{}) + if err != nil { + return nil, err + } + + c, err := federation_clientset.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("error creating federation clientset: %v", err.Error()) + } + return c, nil +} + +func LoadFederatedConfig(overrides *clientcmd.ConfigOverrides) (*restclient.Config, error) { + c, err := framework.RestclientConfig(framework.TestContext.FederatedKubeContext) + if err != nil { + return nil, fmt.Errorf("error creating federation client config: %v", err.Error()) + } + cfg, err := clientcmd.NewDefaultClientConfig(*c, overrides).ClientConfig() + if cfg != nil { + //TODO(colhom): this is only here because https://github.com/kubernetes/kubernetes/issues/25422 + cfg.NegotiatedSerializer = api.Codecs + } + if err != nil { + return cfg, fmt.Errorf("error creating federation default client config: %v", err.Error()) + } + return cfg, nil +} + +// GetValidDNSSubdomainName massages the given name to be a valid dns subdomain name. +// Most resources (such as secrets, clusters) require the names to be valid dns subdomain. +// This is a generic function (not specific to federation). Should be moved to a more generic location if others want to use it. +func GetValidDNSSubdomainName(name string) (string, error) { + // "_" are not allowed. Replace them by "-". + name = regexp.MustCompile("_").ReplaceAllLiteralString(name, "-") + maxLength := validationutil.DNS1123SubdomainMaxLength + if len(name) > maxLength { + name = name[0 : maxLength-1] + } + // Verify that name now passes the validation. + if errors := validation.NameIsDNSSubdomain(name, false); len(errors) != 0 { + return "", fmt.Errorf("errors in converting name to a valid DNS subdomain %s", errors) + } + return name, nil +}