Add CSI volume resizing tests

Add some tests for checking node expansion
Add new tests for expander

test/e2e/storage/csi_mock_volume.go

@@ -28,6 +28,7 @@ import (
 	storage "k8s.io/api/storage/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -50,16 +51,23 @@ type cleanupFuncs func()
 const (
 	csiNodeLimitUpdateTimeout  = 5 * time.Minute
 	csiPodUnschedulableTimeout = 5 * time.Minute
+	csiResizeWaitPeriod        = 5 * time.Minute
+	// how long to wait for Resizing Condition on PVC to appear
+	csiResizingConditionWait = 2 * time.Minute
 )
 
 var _ = utils.SIGDescribe("CSI mock volume", func() {
 	type testParameters struct {
-		disableAttach   bool
-		attachLimit     int
-		registerDriver  bool
-		podInfo         *bool
-		scName          string
-		nodeSelectorKey string
+		disableAttach       bool
+		attachLimit         int
+		registerDriver      bool
+		podInfo             *bool
+		scName              string
+		nodeSelectorKey     string
+		enableResizing      bool // enable resizing for both CSI mock driver and storageClass.
+		enableNodeExpansion bool // enable node expansion for CSI mock driver
+		// just disable resizing on driver it overrides enableResizing flag for CSI mock driver
+		disableResizingOnDriver bool
 	}
 
 	type mockDriverSetup struct {
@@ -87,8 +95,21 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 		}
 		cs := f.ClientSet
 		var err error
+		driverOpts := drivers.CSIMockDriverOpts{
+			RegisterDriver:      tp.registerDriver,
+			PodInfo:             tp.podInfo,
+			AttachLimit:         tp.attachLimit,
+			DisableAttach:       tp.disableAttach,
+			EnableResizing:      tp.enableResizing,
+			EnableNodeExpansion: tp.enableNodeExpansion,
+		}
 
-		m.driver = drivers.InitMockCSIDriver(tp.registerDriver, !tp.disableAttach, tp.podInfo, tp.attachLimit)
+		// this just disable resizing on driver, keeping resizing on SC enabled.
+		if tp.disableResizingOnDriver {
+			driverOpts.EnableResizing = false
+		}
+
+		m.driver = drivers.InitMockCSIDriver(driverOpts)
 		config, testCleanup := m.driver.PrepareTest(f)
 		m.testCleanups = append(m.testCleanups, testCleanup)
 		m.config = config
@@ -127,6 +148,11 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 		if m.tp.scName != "" {
 			scTest.StorageClassName = m.tp.scName
 		}
+
+		if m.tp.enableResizing {
+			scTest.AllowVolumeExpansion = true
+		}
+
 		nodeSelection := testsuites.NodeSelection{
 			// The mock driver only works when everything runs on a single node.
 			Name: nodeName,
@@ -149,6 +175,23 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 		return class, claim, pod
 	}
 
+	createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
+		nodeName := m.config.ClientNodeName
+		nodeSelection := testsuites.NodeSelection{
+			Name: nodeName,
+		}
+		if len(m.nodeLabel) > 0 {
+			nodeSelection = testsuites.NodeSelection{
+				Selector: m.nodeLabel,
+			}
+		}
+		pod, err := startPausePodWithClaim(m.cs, pvc, nodeSelection, f.Namespace.Name)
+		if pod != nil {
+			m.pods = append(m.pods, pod)
+		}
+		return pod, err
+	}
+
 	cleanup := func() {
 		cs := f.ClientSet
 		var errs []error
@@ -340,6 +383,177 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 		})
 	})
 
+	Context("CSI Volume expansion [Feature:ExpandCSIVolumes]", func() {
+		tests := []struct {
+			name                    string
+			nodeExpansionRequired   bool
+			disableAttach           bool
+			disableResizingOnDriver bool
+			expectFailure           bool
+		}{
+			{
+				name:                  "should expand volume without restarting pod if nodeExpansion=off",
+				nodeExpansionRequired: false,
+			},
+			{
+				name:                  "should expand volume by restarting pod if attach=on, nodeExpansion=on",
+				nodeExpansionRequired: true,
+			},
+			{
+				name:                  "should expand volume by restarting pod if attach=off, nodeExpansion=on",
+				disableAttach:         true,
+				nodeExpansionRequired: true,
+			},
+			{
+				name:                    "should not expand volume if resizingOnDriver=off, resizingOnSC=on",
+				disableResizingOnDriver: true,
+				expectFailure:           true,
+			},
+		}
+		for _, t := range tests {
+			test := t
+			It(t.name, func() {
+				var err error
+				tp := testParameters{
+					enableResizing:          true,
+					enableNodeExpansion:     test.nodeExpansionRequired,
+					disableResizingOnDriver: test.disableResizingOnDriver,
+				}
+				// disabling attach requires drive registration feature
+				if test.disableAttach {
+					tp.disableAttach = true
+					tp.registerDriver = true
+				}
+
+				init(tp)
+				defer cleanup()
+
+				ns := f.Namespace.Name
+				sc, pvc, pod := createPod()
+				Expect(pod).NotTo(BeNil(), "while creating pod for resizing")
+
+				Expect(*sc.AllowVolumeExpansion).To(BeTrue(), "failed creating sc with allowed expansion")
+
+				err = framework.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace)
+				framework.ExpectNoError(err, "Failed to start pod1: %v", err)
+
+				By("Expanding current pvc")
+				newSize := resource.MustParse("6Gi")
+				pvc, err = expandPVCSize(pvc, newSize, m.cs)
+				Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size")
+				Expect(pvc).NotTo(BeNil())
+
+				pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
+				if pvcSize.Cmp(newSize) != 0 {
+					framework.Failf("error updating pvc size %q", pvc.Name)
+				}
+				if test.expectFailure {
+					err = waitForResizingCondition(pvc, m.cs, csiResizingConditionWait)
+					Expect(err).To(HaveOccurred(), "unexpected resizing condition on PVC")
+					return
+				}
+
+				By("Waiting for persistent volume resize to finish")
+				err = waitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod)
+				Expect(err).NotTo(HaveOccurred(), "While waiting for CSI PV resize to finish")
+
+				checkPVCSize := func() {
+					By("Waiting for PVC resize to finish")
+					pvc, err = waitForFSResize(pvc, m.cs)
+					Expect(err).NotTo(HaveOccurred(), "while waiting for PVC resize to finish")
+
+					pvcConditions := pvc.Status.Conditions
+					Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions")
+				}
+
+				// if node expansion is not required PVC should be resized as well
+				if !test.nodeExpansionRequired {
+					checkPVCSize()
+				} else {
+					By("Checking for conditions on pvc")
+					pvc, err = m.cs.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
+					Expect(err).NotTo(HaveOccurred(), "While fetching pvc after controller resize")
+
+					inProgressConditions := pvc.Status.Conditions
+					if len(inProgressConditions) > 0 {
+						Expect(inProgressConditions[0].Type).To(Equal(v1.PersistentVolumeClaimFileSystemResizePending), "pvc must have fs resizing condition")
+					}
+
+					By("Deleting the previously created pod")
+					err = framework.DeletePodWithWait(f, m.cs, pod)
+					Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing")
+
+					By("Creating a new pod with same volume")
+					pod2, err := createPodWithPVC(pvc)
+					Expect(pod2).NotTo(BeNil(), "while creating pod for csi resizing")
+					Expect(err).NotTo(HaveOccurred(), "while recreating pod for resizing")
+
+					checkPVCSize()
+				}
+			})
+		}
+	})
+	Context("CSI online volume expansion [Feature:ExpandCSIVolumes][Feature:ExpandInUseVolumes]", func() {
+		tests := []struct {
+			name          string
+			disableAttach bool
+		}{
+			{
+				name: "should expand volume without restarting pod if attach=on, nodeExpansion=on",
+			},
+			{
+				name:          "should expand volume without restarting pod if attach=off, nodeExpansion=on",
+				disableAttach: true,
+			},
+		}
+		for _, t := range tests {
+			test := t
+			It(test.name, func() {
+				var err error
+				params := testParameters{enableResizing: true, enableNodeExpansion: true}
+				if test.disableAttach {
+					params.disableAttach = true
+					params.registerDriver = true
+				}
+
+				init(params)
+
+				defer cleanup()
+
+				sc, pvc, pod := createPod()
+				Expect(pod).NotTo(BeNil(), "while creating pod for resizing")
+
+				Expect(*sc.AllowVolumeExpansion).To(BeTrue(), "failed creating sc with allowed expansion")
+
+				err = framework.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace)
+				framework.ExpectNoError(err, "Failed to start pod1: %v", err)
+
+				By("Expanding current pvc")
+				newSize := resource.MustParse("6Gi")
+				pvc, err = expandPVCSize(pvc, newSize, m.cs)
+				Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size")
+				Expect(pvc).NotTo(BeNil())
+
+				pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
+				if pvcSize.Cmp(newSize) != 0 {
+					framework.Failf("error updating pvc size %q", pvc.Name)
+				}
+
+				By("Waiting for persistent volume resize to finish")
+				err = waitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod)
+				Expect(err).NotTo(HaveOccurred(), "While waiting for PV resize to finish")
+
+				By("Waiting for PVC resize to finish")
+				pvc, err = waitForFSResize(pvc, m.cs)
+				Expect(err).NotTo(HaveOccurred(), "while waiting for PVC to finish")
+
+				pvcConditions := pvc.Status.Conditions
+				Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions")
+
+			})
+		}
+	})
+
 })
 
 func waitForMaxVolumeCondition(pod *v1.Pod, cs clientset.Interface) error {
@@ -445,6 +659,49 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node t
 	return class, claim, pod
 }
 
+func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node testsuites.NodeSelection, ns string) (*v1.Pod, error) {
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-volume-tester-",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "volume-tester",
+					Image: imageutils.GetE2EImage(imageutils.Pause),
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "my-volume",
+							MountPath: "/mnt/test",
+						},
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+			Volumes: []v1.Volume{
+				{
+					Name: "my-volume",
+					VolumeSource: v1.VolumeSource{
+						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+							ClaimName: pvc.Name,
+							ReadOnly:  false,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if node.Name != "" {
+		pod.Spec.NodeName = node.Name
+	}
+	if len(node.Selector) != 0 {
+		pod.Spec.NodeSelector = node.Selector
+	}
+
+	return cs.CoreV1().Pods(ns).Create(pod)
+}
+
 // checkPodInfo tests that NodePublish was called with expected volume_context
 func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo bool) error {
 	expectedAttributes := map[string]string{
@@ -508,7 +765,7 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai
 }
 
 func waitForCSIDriver(cs clientset.Interface, driverName string) error {
-	timeout := 2 * time.Minute
+	timeout := 4 * time.Minute
 
 	framework.Logf("waiting up to %v for CSIDriver %q", timeout, driverName)
 	for start := time.Now(); time.Since(start) < timeout; time.Sleep(framework.Poll) {

test/e2e/storage/drivers/csi.go

@@ -169,36 +169,52 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.Per
 
 // mockCSI
 type mockCSIDriver struct {
-	driverInfo  testsuites.DriverInfo
-	manifests   []string
-	podInfo     *bool
-	attachable  bool
-	attachLimit int
+	driverInfo          testsuites.DriverInfo
+	manifests           []string
+	podInfo             *bool
+	attachable          bool
+	attachLimit         int
+	enableNodeExpansion bool
+}
+
+// CSIMockDriverOpts defines options used for csi driver
+type CSIMockDriverOpts struct {
+	RegisterDriver      bool
+	DisableAttach       bool
+	PodInfo             *bool
+	AttachLimit         int
+	EnableResizing      bool
+	EnableNodeExpansion bool
 }
 
 var _ testsuites.TestDriver = &mockCSIDriver{}
 var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{}
 
 // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface
-func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfo *bool, attachLimit int) testsuites.TestDriver {
+func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver {
 	driverManifests := []string{
 		"test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml",
 		"test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml",
 	}
 
-	if registerDriver {
+	if driverOpts.RegisterDriver {
 		driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml")
 	}
 
-	if driverAttachable {
+	if !driverOpts.DisableAttach {
 		driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml")
 	}
 
+	if driverOpts.EnableResizing {
+		driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-resizer.yaml")
+	}
+
 	return &mockCSIDriver{
 		driverInfo: testsuites.DriverInfo{
 			Name:        "csi-mock",
@@ -213,10 +229,11 @@ func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfo *bool, att
 				testsuites.CapExec:        false,
 			},
 		},
-		manifests:   driverManifests,
-		podInfo:     podInfo,
-		attachable:  driverAttachable,
-		attachLimit: attachLimit,
+		manifests:           driverManifests,
+		podInfo:             driverOpts.PodInfo,
+		attachable:          !driverOpts.DisableAttach,
+		attachLimit:         driverOpts.AttachLimit,
+		enableNodeExpansion: driverOpts.EnableNodeExpansion,
 	}
 }
 
@@ -264,6 +281,10 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest
 		containerArgs = append(containerArgs, "--attach-limit", strconv.Itoa(m.attachLimit))
 	}
 
+	if m.enableNodeExpansion {
+		containerArgs = append(containerArgs, "--node-expand-required=true")
+	}
+
 	// TODO (?): the storage.csi.image.version and storage.csi.image.registry
 	// settings are ignored for this test. We could patch the image definitions.
 	o := utils.PatchCSIOptions{

test/e2e/storage/flexvolume_mounted_volume_resize.go

@@ -159,7 +159,7 @@ var _ = utils.SIGDescribe("Mounted flexvolume expand[Slow]", func() {
 		}
 
 		By("Waiting for cloudprovider resize to finish")
-		err = waitForControllerVolumeResize(pvc, c)
+		err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod)
 		Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish")
 
 		By("Getting a pod from deployment")

test/e2e/storage/flexvolume_online_resize.go

@@ -164,7 +164,7 @@ var _ = utils.SIGDescribe("Mounted flexvolume volume expand [Slow] [Feature:Expa
 		}
 
 		By("Waiting for cloudprovider resize to finish")
-		err = waitForControllerVolumeResize(pvc, c)
+		err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod)
 		Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish")
 
 		By("Waiting for file system resize to finish")

test/e2e/storage/mounted_volume_resize.go

@@ -136,7 +136,7 @@ var _ = utils.SIGDescribe("Mounted volume expand", func() {
 		}
 
 		By("Waiting for cloudprovider resize to finish")
-		err = waitForControllerVolumeResize(pvc, c)
+		err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod)
 		Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish")
 
 		By("Getting a pod from deployment")

test/e2e/storage/volume_expand.go

@@ -136,7 +136,7 @@ var _ = utils.SIGDescribe("Volume expand", func() {
 		}
 
 		By("Waiting for cloudprovider resize to finish")
-		err = waitForControllerVolumeResize(pvc, c)
+		err = waitForControllerVolumeResize(pvc, c, totalResizeWaitPeriod)
 		Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish")
 
 		By("Checking for conditions on pvc")
@@ -198,9 +198,29 @@ func expandPVCSize(origPVC *v1.PersistentVolumeClaim, size resource.Quantity, c
 	return updatedPVC, waitErr
 }
 
-func waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface) error {
+func waitForResizingCondition(pvc *v1.PersistentVolumeClaim, c clientset.Interface, duration time.Duration) error {
+	waitErr := wait.PollImmediate(resizePollInterval, duration, func() (bool, error) {
+		var err error
+		updatedPVC, err := c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
+
+		if err != nil {
+			return false, fmt.Errorf("error fetching pvc %q for checking for resize status : %v", pvc.Name, err)
+		}
+
+		pvcConditions := updatedPVC.Status.Conditions
+		if len(pvcConditions) > 0 {
+			if pvcConditions[0].Type == v1.PersistentVolumeClaimResizing {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+	return waitErr
+}
+
+func waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface, duration time.Duration) error {
 	pvName := pvc.Spec.VolumeName
-	return wait.PollImmediate(resizePollInterval, totalResizeWaitPeriod, func() (bool, error) {
+	return wait.PollImmediate(resizePollInterval, duration, func() (bool, error) {
 		pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
 
 		pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})