github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix handle terminating pvc when kubelet rebuild dsw

Browse Source 
Signed-off-by: carlory <baofa.fan@daocloud.io>
This commit is contained in:
carlory
2025-02-21 18:36:06 +08:00
parent 77de0b99c2
commit 3836d58744
2 changed files with 31 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
test/e2e/storage/csimock/csi_kubelet_restart.go
View File

@@ -19,27 +19,35 @@ package csimock
import (
"context"
"fmt"
"os"
"os/exec"
"strings"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
"k8s.io/kubernetes/test/e2e/storage/drivers"
"k8s.io/kubernetes/test/e2e/storage/utils"
admissionapi "k8s.io/pod-security-admission/api"
)
var _ = utils.SIGDescribe("CSI Mock when kubelet restart", feature.Kind, framework.WithSerial(), framework.WithDisruptive(), func() {
var _ = utils.SIGDescribe("CSI Mock when kubelet restart", framework.WithSerial(), framework.WithDisruptive(), func() {
f := framework.NewDefaultFramework("csi-mock-when-kubelet-restart")
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
m := newMockDriverSetup(f)
ginkgo.BeforeEach(func() {
// These tests requires SSH to nodes, so the provider check should be identical to there
// (the limiting factor is the implementation of util.go's e2essh.GetSigner(...)).
// Cluster must support node reboot
e2eskipper.SkipUnlessProviderIs(framework.ProvidersWithSSH...)
e2eskipper.SkipUnlessSSHKeyPresent()
})
ginkgo.It("should not umount volume when the pvc is terminating but still used by a running pod", func(ctx context.Context) {
m.init(ctx, testParameters{
registerDriver: true,
})
@@ -51,16 +59,16 @@ var _ = utils.SIGDescribe("CSI Mock when kubelet restart", feature.Kind, framewo
ginkgo.By("Waiting for the Pod to be running")
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "failed to wait for pod %s to be running", pod.Name)
pod, err = f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get pod %s", pod.Name)
ginkgo.By("Deleting the PVC")
err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(ctx, pvc.Name, metav1.DeleteOptions{})
framework.ExpectNoError(err, "failed to delete PVC %s", pvc.Name)
ginkgo.By("Restarting kubelet")
err = stopKindKubelet(ctx)
framework.ExpectNoError(err, "failed to stop kubelet")
err = startKindKubelet(ctx)
framework.ExpectNoError(err, "failed to start kubelet")
utils.KubeletCommand(ctx, utils.KRestart, f.ClientSet, pod)
ginkgo.DeferCleanup(utils.KubeletCommand, utils.KStart, f.ClientSet, pod)
ginkgo.By("Verifying the PVC is terminating during kubelet restart")
pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{})
@@ -69,7 +77,7 @@ var _ = utils.SIGDescribe("CSI Mock when kubelet restart", feature.Kind, framewo
ginkgo.By(fmt.Sprintf("Verifying that the driver didn't receive NodeUnpublishVolume call for PVC %s", pvc.Name))
gomega.Consistently(ctx,
func(ctx context.Context) interface{} {
func(ctx context.Context) []drivers.MockCSICall {
calls, err := m.driver.GetCalls(ctx)
if err != nil {
if apierrors.IsUnexpectedServerError(err) {
@@ -90,39 +98,3 @@ var _ = utils.SIGDescribe("CSI Mock when kubelet restart", feature.Kind, framewo
framework.ExpectNoError(err, "failed to wait for pod %s to be running", pod.Name)
})
})
func stopKindKubelet(ctx context.Context) error {
return kubeletExec("systemctl", "stop", "kubelet")
}
func startKindKubelet(ctx context.Context) error {
return kubeletExec("systemctl", "start", "kubelet")
}
// Run a command in container with kubelet (and the whole control plane as containers)
func kubeletExec(command ...string) error {
containerName := getKindContainerName()
args := []string{"exec", containerName}
args = append(args, command...)
cmd := exec.Command("docker", args...)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("command %q failed: %v\noutput:%s", prettyCmd(cmd), err, string(out))
}
framework.Logf("command %q succeeded:\n%s", prettyCmd(cmd), string(out))
return nil
}
func getKindContainerName() string {
clusterName := os.Getenv("KIND_CLUSTER_NAME")
if clusterName == "" {
clusterName = "kind"
}
return clusterName + "-control-plane"
}
func prettyCmd(cmd *exec.Cmd) string {
return fmt.Sprintf("%s %s", cmd.Path, strings.Join(cmd.Args, " "))
}