Merge pull request #120977 from borg-land/remove-kubeup-tests-2
Remove an e2e test specific to kubeup clusters - Part Two
This commit is contained in:
Kubernetes Prow Robot
@@ -1,522 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2015 The Kubernetes Authors.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package gcp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"golang.org/x/crypto/ssh"
|
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
||||||
"k8s.io/apimachinery/pkg/labels"
|
|
||||||
"k8s.io/apimachinery/pkg/util/wait"
|
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
|
||||||
e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
|
|
||||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
|
||||||
e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
|
|
||||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
|
||||||
admissionapi "k8s.io/pod-security-admission/api"
|
|
||||||
|
|
||||||
"github.com/onsi/ginkgo/v2"
|
|
||||||
"github.com/onsi/gomega"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TODO: it would probably be slightly better to build up the objects
|
|
||||||
// in the code and then serialize to yaml.
|
|
||||||
var reconcileAddonController = `
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ReplicationController
|
|
||||||
metadata:
|
|
||||||
name: addon-reconcile-test
|
|
||||||
namespace: %s
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-reconcile-test
|
|
||||||
kubernetes.io/cluster-service: "true"
|
|
||||||
addonmanager.kubernetes.io/mode: Reconcile
|
|
||||||
spec:
|
|
||||||
replicas: 2
|
|
||||||
selector:
|
|
||||||
k8s-app: addon-reconcile-test
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-reconcile-test
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- image: %s
|
|
||||||
name: addon-reconcile-test
|
|
||||||
ports:
|
|
||||||
- containerPort: 9376
|
|
||||||
protocol: TCP
|
|
||||||
`
|
|
||||||
|
|
||||||
// Should update "reconcile" class addon.
|
|
||||||
var reconcileAddonControllerUpdated = `
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ReplicationController
|
|
||||||
metadata:
|
|
||||||
name: addon-reconcile-test
|
|
||||||
namespace: %s
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-reconcile-test
|
|
||||||
kubernetes.io/cluster-service: "true"
|
|
||||||
addonmanager.kubernetes.io/mode: Reconcile
|
|
||||||
newLabel: addon-reconcile-test
|
|
||||||
spec:
|
|
||||||
replicas: 2
|
|
||||||
selector:
|
|
||||||
k8s-app: addon-reconcile-test
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-reconcile-test
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- image: %s
|
|
||||||
name: addon-reconcile-test
|
|
||||||
ports:
|
|
||||||
- containerPort: 9376
|
|
||||||
protocol: TCP
|
|
||||||
`
|
|
||||||
|
|
||||||
var ensureExistsAddonService = `
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: addon-ensure-exists-test
|
|
||||||
namespace: %s
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-ensure-exists-test
|
|
||||||
addonmanager.kubernetes.io/mode: EnsureExists
|
|
||||||
spec:
|
|
||||||
ports:
|
|
||||||
- port: 9376
|
|
||||||
protocol: TCP
|
|
||||||
targetPort: 9376
|
|
||||||
selector:
|
|
||||||
k8s-app: addon-ensure-exists-test
|
|
||||||
`
|
|
||||||
|
|
||||||
// Should create but don't update "ensure exist" class addon.
|
|
||||||
var ensureExistsAddonServiceUpdated = `
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: addon-ensure-exists-test
|
|
||||||
namespace: %s
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-ensure-exists-test
|
|
||||||
addonmanager.kubernetes.io/mode: EnsureExists
|
|
||||||
newLabel: addon-ensure-exists-test
|
|
||||||
spec:
|
|
||||||
ports:
|
|
||||||
- port: 9376
|
|
||||||
protocol: TCP
|
|
||||||
targetPort: 9376
|
|
||||||
selector:
|
|
||||||
k8s-app: addon-ensure-exists-test
|
|
||||||
`
|
|
||||||
|
|
||||||
var deprecatedLabelAddonService = `
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: addon-deprecated-label-test
|
|
||||||
namespace: %s
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-deprecated-label-test
|
|
||||||
kubernetes.io/cluster-service: "true"
|
|
||||||
spec:
|
|
||||||
ports:
|
|
||||||
- port: 9376
|
|
||||||
protocol: TCP
|
|
||||||
targetPort: 9376
|
|
||||||
selector:
|
|
||||||
k8s-app: addon-deprecated-label-test
|
|
||||||
`
|
|
||||||
|
|
||||||
// Should update addon with label "kubernetes.io/cluster-service=true".
|
|
||||||
var deprecatedLabelAddonServiceUpdated = `
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: addon-deprecated-label-test
|
|
||||||
namespace: %s
|
|
||||||
labels:
|
|
||||||
k8s-app: addon-deprecated-label-test
|
|
||||||
kubernetes.io/cluster-service: "true"
|
|
||||||
newLabel: addon-deprecated-label-test
|
|
||||||
spec:
|
|
||||||
ports:
|
|
||||||
- port: 9376
|
|
||||||
protocol: TCP
|
|
||||||
targetPort: 9376
|
|
||||||
selector:
|
|
||||||
k8s-app: addon-deprecated-label-test
|
|
||||||
`
|
|
||||||
|
|
||||||
// Should not create addon without valid label.
|
|
||||||
var invalidAddonController = `
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ReplicationController
|
|
||||||
metadata:
|
|
||||||
name: invalid-addon-test
|
|
||||||
namespace: %s
|
|
||||||
labels:
|
|
||||||
k8s-app: invalid-addon-test
|
|
||||||
addonmanager.kubernetes.io/mode: NotMatch
|
|
||||||
spec:
|
|
||||||
replicas: 2
|
|
||||||
selector:
|
|
||||||
k8s-app: invalid-addon-test
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
k8s-app: invalid-addon-test
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- image: %s
|
|
||||||
name: invalid-addon-test
|
|
||||||
ports:
|
|
||||||
- containerPort: 9376
|
|
||||||
protocol: TCP
|
|
||||||
`
|
|
||||||
|
|
||||||
const (
|
|
||||||
addonTestPollInterval = 3 * time.Second
|
|
||||||
addonTestPollTimeout = 5 * time.Minute
|
|
||||||
addonNsName = metav1.NamespaceSystem
|
|
||||||
)
|
|
||||||
|
|
||||||
var serveHostnameImage = imageutils.GetE2EImage(imageutils.Agnhost)
|
|
||||||
|
|
||||||
type stringPair struct {
|
|
||||||
data, fileName string
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ = SIGDescribe("Addon update", func() {
|
|
||||||
|
|
||||||
var dir string
|
|
||||||
var sshClient *ssh.Client
|
|
||||||
f := framework.NewDefaultFramework("addon-update-test")
|
|
||||||
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
|
|
||||||
|
|
||||||
ginkgo.BeforeEach(func() {
|
|
||||||
// This test requires:
|
|
||||||
// - SSH master access
|
|
||||||
// ... so the provider check should be identical to the intersection of
|
|
||||||
// providers that provide those capabilities.
|
|
||||||
if !framework.ProviderIs("gce") {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var err error
|
|
||||||
sshClient, err = getMasterSSHClient()
|
|
||||||
framework.ExpectNoError(err, "Failed to get the master SSH client.")
|
|
||||||
})
|
|
||||||
|
|
||||||
ginkgo.AfterEach(func() {
|
|
||||||
if sshClient != nil {
|
|
||||||
sshClient.Close()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// WARNING: the test is not parallel-friendly!
|
|
||||||
ginkgo.It("should propagate add-on file changes [Slow]", func(ctx context.Context) {
|
|
||||||
// This test requires:
|
|
||||||
// - SSH
|
|
||||||
// - master access
|
|
||||||
// ... so the provider check should be identical to the intersection of
|
|
||||||
// providers that provide those capabilities.
|
|
||||||
e2eskipper.SkipUnlessProviderIs("gce")
|
|
||||||
|
|
||||||
//these tests are long, so I squeezed several cases in one scenario
|
|
||||||
framework.ExpectNotEqual(sshClient, nil)
|
|
||||||
dir = f.Namespace.Name // we use it only to give a unique string for each test execution
|
|
||||||
|
|
||||||
temporaryRemotePathPrefix := "addon-test-dir"
|
|
||||||
temporaryRemotePath := temporaryRemotePathPrefix + "/" + dir // in home directory on kubernetes-master
|
|
||||||
defer sshExec(sshClient, fmt.Sprintf("rm -rf %s", temporaryRemotePathPrefix)) // ignore the result in cleanup
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("mkdir -p %s", temporaryRemotePath))
|
|
||||||
|
|
||||||
rcAddonReconcile := "addon-reconcile-controller.yaml"
|
|
||||||
rcAddonReconcileUpdated := "addon-reconcile-controller-Updated.yaml"
|
|
||||||
rcInvalid := "invalid-addon-controller.yaml"
|
|
||||||
|
|
||||||
svcAddonDeprecatedLabel := "addon-deprecated-label-service.yaml"
|
|
||||||
svcAddonDeprecatedLabelUpdated := "addon-deprecated-label-service-updated.yaml"
|
|
||||||
svcAddonEnsureExists := "addon-ensure-exists-service.yaml"
|
|
||||||
svcAddonEnsureExistsUpdated := "addon-ensure-exists-service-updated.yaml"
|
|
||||||
|
|
||||||
var remoteFiles = []stringPair{
|
|
||||||
{fmt.Sprintf(reconcileAddonController, addonNsName, serveHostnameImage), rcAddonReconcile},
|
|
||||||
{fmt.Sprintf(reconcileAddonControllerUpdated, addonNsName, serveHostnameImage), rcAddonReconcileUpdated},
|
|
||||||
{fmt.Sprintf(deprecatedLabelAddonService, addonNsName), svcAddonDeprecatedLabel},
|
|
||||||
{fmt.Sprintf(deprecatedLabelAddonServiceUpdated, addonNsName), svcAddonDeprecatedLabelUpdated},
|
|
||||||
{fmt.Sprintf(ensureExistsAddonService, addonNsName), svcAddonEnsureExists},
|
|
||||||
{fmt.Sprintf(ensureExistsAddonServiceUpdated, addonNsName), svcAddonEnsureExistsUpdated},
|
|
||||||
{fmt.Sprintf(invalidAddonController, addonNsName, serveHostnameImage), rcInvalid},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, p := range remoteFiles {
|
|
||||||
err := writeRemoteFile(sshClient, p.data, temporaryRemotePath, p.fileName, 0644)
|
|
||||||
framework.ExpectNoError(err, "Failed to write file %q at remote path %q with ssh client %+v", p.fileName, temporaryRemotePath, sshClient)
|
|
||||||
}
|
|
||||||
|
|
||||||
// directory on kubernetes-master
|
|
||||||
destinationDirPrefix := "/etc/kubernetes/addons/addon-test-dir"
|
|
||||||
destinationDir := destinationDirPrefix + "/" + dir
|
|
||||||
|
|
||||||
// cleanup from previous tests
|
|
||||||
_, _, _, err := sshExec(sshClient, fmt.Sprintf("sudo rm -rf %s", destinationDirPrefix))
|
|
||||||
framework.ExpectNoError(err, "Failed to remove remote dir %q with ssh client %+v", destinationDirPrefix, sshClient)
|
|
||||||
|
|
||||||
defer sshExec(sshClient, fmt.Sprintf("sudo rm -rf %s", destinationDirPrefix)) // ignore result in cleanup
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo mkdir -p %s", destinationDir))
|
|
||||||
|
|
||||||
ginkgo.By("copy invalid manifests to the destination dir")
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcInvalid, destinationDir, rcInvalid))
|
|
||||||
// we will verify at the end of the test that the objects weren't created from the invalid manifests
|
|
||||||
|
|
||||||
ginkgo.By("copy new manifests")
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcAddonReconcile, destinationDir, rcAddonReconcile))
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonDeprecatedLabel, destinationDir, svcAddonDeprecatedLabel))
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonEnsureExists, destinationDir, svcAddonEnsureExists))
|
|
||||||
// Delete the "ensure exist class" addon at the end.
|
|
||||||
defer func() {
|
|
||||||
framework.Logf("Cleaning up ensure exist class addon.")
|
|
||||||
err := f.ClientSet.CoreV1().Services(addonNsName).Delete(ctx, "addon-ensure-exists-test", metav1.DeleteOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
}()
|
|
||||||
|
|
||||||
waitForReplicationControllerInAddonTest(ctx, f.ClientSet, addonNsName, "addon-reconcile-test", true)
|
|
||||||
waitForServiceInAddonTest(ctx, f.ClientSet, addonNsName, "addon-deprecated-label-test", true)
|
|
||||||
waitForServiceInAddonTest(ctx, f.ClientSet, addonNsName, "addon-ensure-exists-test", true)
|
|
||||||
|
|
||||||
// Replace the manifests with new contents.
|
|
||||||
ginkgo.By("update manifests")
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, rcAddonReconcileUpdated, destinationDir, rcAddonReconcile))
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonDeprecatedLabelUpdated, destinationDir, svcAddonDeprecatedLabel))
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo cp %s/%s %s/%s", temporaryRemotePath, svcAddonEnsureExistsUpdated, destinationDir, svcAddonEnsureExists))
|
|
||||||
|
|
||||||
// Wait for updated addons to have the new added label.
|
|
||||||
reconcileSelector := labels.SelectorFromSet(labels.Set(map[string]string{"newLabel": "addon-reconcile-test"}))
|
|
||||||
waitForReplicationControllerwithSelectorInAddonTest(ctx, f.ClientSet, addonNsName, true, reconcileSelector)
|
|
||||||
deprecatedLabelSelector := labels.SelectorFromSet(labels.Set(map[string]string{"newLabel": "addon-deprecated-label-test"}))
|
|
||||||
waitForServicewithSelectorInAddonTest(ctx, f.ClientSet, addonNsName, true, deprecatedLabelSelector)
|
|
||||||
// "Ensure exist class" addon should not be updated.
|
|
||||||
ensureExistSelector := labels.SelectorFromSet(labels.Set(map[string]string{"newLabel": "addon-ensure-exists-test"}))
|
|
||||||
waitForServicewithSelectorInAddonTest(ctx, f.ClientSet, addonNsName, false, ensureExistSelector)
|
|
||||||
|
|
||||||
ginkgo.By("remove manifests")
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, rcAddonReconcile))
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcAddonDeprecatedLabel))
|
|
||||||
sshExecAndVerify(sshClient, fmt.Sprintf("sudo rm %s/%s", destinationDir, svcAddonEnsureExists))
|
|
||||||
|
|
||||||
waitForReplicationControllerInAddonTest(ctx, f.ClientSet, addonNsName, "addon-reconcile-test", false)
|
|
||||||
waitForServiceInAddonTest(ctx, f.ClientSet, addonNsName, "addon-deprecated-label-test", false)
|
|
||||||
// "Ensure exist class" addon will not be deleted when manifest is removed.
|
|
||||||
waitForServiceInAddonTest(ctx, f.ClientSet, addonNsName, "addon-ensure-exists-test", true)
|
|
||||||
|
|
||||||
ginkgo.By("verify invalid addons weren't created")
|
|
||||||
_, err = f.ClientSet.CoreV1().ReplicationControllers(addonNsName).Get(ctx, "invalid-addon-test", metav1.GetOptions{})
|
|
||||||
gomega.Expect(err).To(gomega.HaveOccurred())
|
|
||||||
|
|
||||||
// Invalid addon manifests and the "ensure exist class" addon will be deleted by the deferred function.
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
func waitForServiceInAddonTest(ctx context.Context, c clientset.Interface, addonNamespace, name string, exist bool) {
|
|
||||||
framework.ExpectNoError(e2enetwork.WaitForService(ctx, c, addonNamespace, name, exist, addonTestPollInterval, addonTestPollTimeout))
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForReplicationControllerInAddonTest(ctx context.Context, c clientset.Interface, addonNamespace, name string, exist bool) {
|
|
||||||
framework.ExpectNoError(waitForReplicationController(ctx, c, addonNamespace, name, exist, addonTestPollInterval, addonTestPollTimeout))
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForServicewithSelectorInAddonTest(ctx context.Context, c clientset.Interface, addonNamespace string, exist bool, selector labels.Selector) {
|
|
||||||
framework.ExpectNoError(waitForServiceWithSelector(ctx, c, addonNamespace, selector, exist, addonTestPollInterval, addonTestPollTimeout))
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForReplicationControllerwithSelectorInAddonTest(ctx context.Context, c clientset.Interface, addonNamespace string, exist bool, selector labels.Selector) {
|
|
||||||
framework.ExpectNoError(waitForReplicationControllerWithSelector(ctx, c, addonNamespace, selector, exist, addonTestPollInterval,
|
|
||||||
addonTestPollTimeout))
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForReplicationController waits until the RC appears (exist == true), or disappears (exist == false)
|
|
||||||
func waitForReplicationController(ctx context.Context, c clientset.Interface, namespace, name string, exist bool, interval, timeout time.Duration) error {
|
|
||||||
err := wait.PollImmediateWithContext(ctx, interval, timeout, func(ctx context.Context) (bool, error) {
|
|
||||||
_, err := c.CoreV1().ReplicationControllers(namespace).Get(ctx, name, metav1.GetOptions{})
|
|
||||||
if err != nil {
|
|
||||||
framework.Logf("Get ReplicationController %s in namespace %s failed (%v).", name, namespace, err)
|
|
||||||
return !exist, nil
|
|
||||||
}
|
|
||||||
framework.Logf("ReplicationController %s in namespace %s found.", name, namespace)
|
|
||||||
return exist, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
stateMsg := map[bool]string{true: "to appear", false: "to disappear"}
|
|
||||||
return fmt.Errorf("error waiting for ReplicationController %s/%s %s: %w", namespace, name, stateMsg[exist], err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForServiceWithSelector waits until any service with given selector appears (exist == true), or disappears (exist == false)
|
|
||||||
func waitForServiceWithSelector(ctx context.Context, c clientset.Interface, namespace string, selector labels.Selector, exist bool, interval,
|
|
||||||
timeout time.Duration) error {
|
|
||||||
err := wait.PollImmediateWithContext(ctx, interval, timeout, func(ctx context.Context) (bool, error) {
|
|
||||||
services, err := c.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
|
|
||||||
switch {
|
|
||||||
case len(services.Items) != 0:
|
|
||||||
framework.Logf("Service with %s in namespace %s found.", selector.String(), namespace)
|
|
||||||
return exist, nil
|
|
||||||
case len(services.Items) == 0:
|
|
||||||
framework.Logf("Service with %s in namespace %s disappeared.", selector.String(), namespace)
|
|
||||||
return !exist, nil
|
|
||||||
case err != nil:
|
|
||||||
framework.Logf("Non-retryable failure while listing service.")
|
|
||||||
return false, err
|
|
||||||
default:
|
|
||||||
framework.Logf("List service with %s in namespace %s failed: %v", selector.String(), namespace, err)
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
stateMsg := map[bool]string{true: "to appear", false: "to disappear"}
|
|
||||||
return fmt.Errorf("error waiting for service with %s in namespace %s %s: %w", selector.String(), namespace, stateMsg[exist], err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForReplicationControllerWithSelector waits until any RC with given selector appears (exist == true), or disappears (exist == false)
|
|
||||||
func waitForReplicationControllerWithSelector(ctx context.Context, c clientset.Interface, namespace string, selector labels.Selector, exist bool, interval,
|
|
||||||
timeout time.Duration) error {
|
|
||||||
err := wait.PollImmediateWithContext(ctx, interval, timeout, func(ctx context.Context) (bool, error) {
|
|
||||||
rcs, err := c.CoreV1().ReplicationControllers(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
|
|
||||||
switch {
|
|
||||||
case len(rcs.Items) != 0:
|
|
||||||
framework.Logf("ReplicationController with %s in namespace %s found.", selector.String(), namespace)
|
|
||||||
return exist, nil
|
|
||||||
case len(rcs.Items) == 0:
|
|
||||||
framework.Logf("ReplicationController with %s in namespace %s disappeared.", selector.String(), namespace)
|
|
||||||
return !exist, nil
|
|
||||||
default:
|
|
||||||
framework.Logf("List ReplicationController with %s in namespace %s failed: %v", selector.String(), namespace, err)
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
stateMsg := map[bool]string{true: "to appear", false: "to disappear"}
|
|
||||||
return fmt.Errorf("error waiting for ReplicationControllers with %s in namespace %s %s: %w", selector.String(), namespace, stateMsg[exist], err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO use the ssh.SSH code, either adding an SCP to it or copying files
|
|
||||||
// differently.
|
|
||||||
func getMasterSSHClient() (*ssh.Client, error) {
|
|
||||||
// Get a signer for the provider.
|
|
||||||
signer, err := e2essh.GetSigner(framework.TestContext.Provider)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error getting signer for provider %s: %w", framework.TestContext.Provider, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sshUser := os.Getenv("KUBE_SSH_USER")
|
|
||||||
if sshUser == "" {
|
|
||||||
sshUser = os.Getenv("USER")
|
|
||||||
}
|
|
||||||
config := &ssh.ClientConfig{
|
|
||||||
User: sshUser,
|
|
||||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
|
||||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
|
||||||
}
|
|
||||||
|
|
||||||
host := framework.APIAddress() + ":22"
|
|
||||||
client, err := ssh.Dial("tcp", host, config)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error getting SSH client to host %s: %w", host, err)
|
|
||||||
}
|
|
||||||
return client, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshExecAndVerify(client *ssh.Client, cmd string) {
|
|
||||||
_, _, rc, err := sshExec(client, cmd)
|
|
||||||
framework.ExpectNoError(err, "Failed to execute %q with ssh client %+v", cmd, client)
|
|
||||||
gomega.Expect(rc).To(gomega.BeZero(), "error return code from executing command on the cluster: %s", cmd)
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshExec(client *ssh.Client, cmd string) (string, string, int, error) {
|
|
||||||
framework.Logf("Executing '%s' on %v", cmd, client.RemoteAddr())
|
|
||||||
session, err := client.NewSession()
|
|
||||||
if err != nil {
|
|
||||||
return "", "", 0, fmt.Errorf("error creating session to host %s: %w", client.RemoteAddr(), err)
|
|
||||||
}
|
|
||||||
defer session.Close()
|
|
||||||
|
|
||||||
// Run the command.
|
|
||||||
code := 0
|
|
||||||
var bout, berr bytes.Buffer
|
|
||||||
|
|
||||||
session.Stdout, session.Stderr = &bout, &berr
|
|
||||||
err = session.Run(cmd)
|
|
||||||
if err != nil {
|
|
||||||
// Check whether the command failed to run or didn't complete.
|
|
||||||
if exiterr, ok := err.(*ssh.ExitError); ok {
|
|
||||||
// If we got an ExitError and the exit code is nonzero, we'll
|
|
||||||
// consider the SSH itself successful (just that the command run
|
|
||||||
// errored on the host).
|
|
||||||
if code = exiterr.ExitStatus(); code != 0 {
|
|
||||||
err = nil
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Some other kind of error happened (e.g. an IOError); consider the
|
|
||||||
// SSH unsuccessful.
|
|
||||||
err = fmt.Errorf("failed running `%s` on %s: %w", cmd, client.RemoteAddr(), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return bout.String(), berr.String(), code, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeRemoteFile(sshClient *ssh.Client, data, dir, fileName string, mode os.FileMode) error {
|
|
||||||
framework.Logf(fmt.Sprintf("Writing remote file '%s/%s' on %v", dir, fileName, sshClient.RemoteAddr()))
|
|
||||||
session, err := sshClient.NewSession()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error creating session to host %s: %w", sshClient.RemoteAddr(), err)
|
|
||||||
}
|
|
||||||
defer session.Close()
|
|
||||||
|
|
||||||
fileSize := len(data)
|
|
||||||
pipe, err := session.StdinPipe()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer pipe.Close()
|
|
||||||
if err := session.Start(fmt.Sprintf("scp -t %s", dir)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Fprintf(pipe, "C%#o %d %s\n", mode, fileSize, fileName)
|
|
||||||
io.Copy(pipe, strings.NewReader(data))
|
|
||||||
fmt.Fprint(pipe, "\x00")
|
|
||||||
pipe.Close()
|
|
||||||
return session.Wait()
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user