github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #124151 from ConnorJC3/add-vac-e2e

Browse Source 
Add Happy Path VolumeAttributesClass CSI E2E Tests
This commit is contained in:
Kubernetes Prow Robot
2024-06-10 18:08:01 -07:00
committed by GitHub
parent c49b140c45 ea58abfd99
commit 9d63e575f8
13 changed files with 483 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
test/e2e/feature/feature.go
View File

@@ -363,6 +363,13 @@ var (
// TODO: document the feature (owning SIG, when to use this feature for a test)
ValidatingAdmissionPolicy = framework.WithFeature(framework.ValidFeatures.Add("ValidatingAdmissionPolicy"))
// Owner: sig-storage
// Tests related to VolumeAttributesClass (https://kep.k8s.io/3751)
//
// TODO: This label only requires the API storage.k8s.io/v1alpha1 and the VolumeAttributesClass feature-gate enabled.
// It should be removed after k/k #124350 is merged.
VolumeAttributesClass = framework.WithFeature(framework.ValidFeatures.Add("VolumeAttributesClass"))
// TODO: document the feature (owning SIG, when to use this feature for a test)
Volumes = framework.WithFeature(framework.ValidFeatures.Add("Volumes"))

14
test/e2e/framework/pv/pv.go
View File

@@ -127,10 +127,11 @@ type PersistentVolumeClaimConfig struct {
// unspecified
ClaimSize string
// AccessModes defaults to RWO if unspecified
AccessModes []v1.PersistentVolumeAccessMode
Annotations map[string]string
Selector *metav1.LabelSelector
StorageClassName *string
AccessModes []v1.PersistentVolumeAccessMode
Annotations map[string]string
Selector *metav1.LabelSelector
StorageClassName *string
VolumeAttributesClassName *string
// VolumeMode defaults to nil if unspecified or specified as the empty
// string
VolumeMode *v1.PersistentVolumeMode
@@ -661,8 +662,9 @@ func MakePersistentVolumeClaim(cfg PersistentVolumeClaimConfig, ns string) *v1.P
v1.ResourceStorage: resource.MustParse(cfg.ClaimSize),
},
},
StorageClassName: cfg.StorageClassName,
VolumeMode: cfg.VolumeMode,
StorageClassName: cfg.StorageClassName,
VolumeAttributesClassName: cfg.VolumeAttributesClassName,
VolumeMode: cfg.VolumeMode,
},
}
}

45
test/e2e/storage/drivers/csi.go
View File

@@ -54,6 +54,7 @@ import (
v1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
storagev1 "k8s.io/api/storage/v1"
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -85,6 +86,11 @@ const (
// Prefix of the mock driver grpc log
grpcCallPrefix = "gRPCCall:"
// Parameter to use in hostpath CSI driver VolumeAttributesClass
// Must be passed to the driver via --accepted-mutable-parameter-names
hostpathCSIDriverMutableParameterName = "e2eVacTest"
hostpathCSIDriverMutableParameterValue = "test-value"
)
// hostpathCSI
@@ -209,6 +215,15 @@ func (h *hostpathCSIDriver) GetSnapshotClass(ctx context.Context, config *storag
return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns)
}
func (h *hostpathCSIDriver) GetVolumeAttributesClass(_ context.Context, config *storageframework.PerTestConfig) *storagev1alpha1.VolumeAttributesClass {
return storageframework.CopyVolumeAttributesClass(&storagev1alpha1.VolumeAttributesClass{
DriverName: config.GetUniqueDriverName(),
Parameters: map[string]string{
hostpathCSIDriverMutableParameterName: hostpathCSIDriverMutableParameterValue,
},
}, config.Framework.Namespace.Name, "e2e-vac-hostpath")
}
func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
// Create secondary namespace which will be used for creating driver
driverNamespace := utils.CreateDriverNamespace(ctx, f)
@@ -230,7 +245,9 @@ func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framew
DriverNamespace: driverNamespace,
}
o := utils.PatchCSIOptions{
patches := []utils.PatchCSIOptions{}
patches = append(patches, utils.PatchCSIOptions{
OldDriverName: h.driverInfo.Name,
NewDriverName: config.GetUniqueDriverName(),
DriverContainerName: "hostpath",
@@ -246,11 +263,31 @@ func (h *hostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framew
ProvisionerContainerName: "csi-provisioner",
SnapshotterContainerName: "csi-snapshotter",
NodeName: node.Name,
}
})
// VAC E2E HostPath patch
// Enables ModifyVolume support in the hostpath CSI driver, and adds an enabled parameter name
patches = append(patches, utils.PatchCSIOptions{
DriverContainerName: "hostpath",
DriverContainerArguments: []string{"--enable-controller-modify-volume=true", "--accepted-mutable-parameter-names=e2eVacTest"},
})
// VAC E2E FeatureGate patches
// TODO: These can be removed after the VolumeAttributesClass feature is default enabled
patches = append(patches, utils.PatchCSIOptions{
DriverContainerName: "csi-provisioner",
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
})
patches = append(patches, utils.PatchCSIOptions{
DriverContainerName: "csi-resizer",
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
})
err = utils.CreateFromManifests(ctx, config.Framework, driverNamespace, func(item interface{}) error {
if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
return err
for _, o := range patches {
if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
return err
}
}
// Remove csi-external-health-monitor-agent and

62
test/e2e/storage/external/external.go vendored
View File

@@ -25,6 +25,7 @@ import (
"time"
storagev1 "k8s.io/api/storage/v1"
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
@@ -79,6 +80,30 @@ type driverDefinition struct {
FromExistingClassName string
}
// VolumeAttributesClass must be set to enable volume modification tests.
// The default is to not run those tests.
VolumeAttributesClass struct {
// FromName set to true enables the usage of a
// VolumeAttributesClass with DriverInfo.Name as
// provisioner and no parameters.
FromName bool
// FromFile is used only when FromName is false. It
// loads a storage class from the given .yaml or .json
// file. File names are resolved by the
// framework.testfiles package, which typically means
// that they can be absolute or relative to the test
// suite's --repo-root parameter.
//
// This can be used when the VolumeAttributesClass
// is meant to have additional parameters.
FromFile string
// FromExistingClassName specifies the name of a pre-installed
// VolumeAttributesClass that will be copied and used for the tests.
FromExistingClassName string
}
// SnapshotClass must be set to enable snapshotting tests.
// The default is to not run those tests.
SnapshotClass struct {
@@ -405,6 +430,43 @@ func (d *driverDefinition) GetSnapshotClass(ctx context.Context, e2econfig *stor
return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns)
}
func (d *driverDefinition) GetVolumeAttributesClass(ctx context.Context, e2econfig *storageframework.PerTestConfig) *storagev1alpha1.VolumeAttributesClass {
if !d.VolumeAttributesClass.FromName && d.VolumeAttributesClass.FromFile == "" && d.VolumeAttributesClass.FromExistingClassName == "" {
e2eskipper.Skipf("Driver %q has no configured VolumeAttributesClass - skipping", d.DriverInfo.Name)
return nil
}
var (
vac *storagev1alpha1.VolumeAttributesClass
err error
)
f := e2econfig.Framework
switch {
case d.VolumeAttributesClass.FromName:
vac = &storagev1alpha1.VolumeAttributesClass{DriverName: d.DriverInfo.Name}
case d.VolumeAttributesClass.FromExistingClassName != "":
vac, err = f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Get(ctx, d.VolumeAttributesClass.FromExistingClassName, metav1.GetOptions{})
framework.ExpectNoError(err, "getting VolumeAttributesClass %s", d.VolumeAttributesClass.FromExistingClassName)
case d.VolumeAttributesClass.FromFile != "":
var ok bool
items, err := utils.LoadFromManifests(d.VolumeAttributesClass.FromFile)
framework.ExpectNoError(err, "load VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile)
gomega.Expect(items).To(gomega.HaveLen(1), "exactly one item from %s", d.VolumeAttributesClass.FromFile)
err = utils.PatchItems(f, f.Namespace, items...)
framework.ExpectNoError(err, "patch VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile)
vac, ok = items[0].(*storagev1alpha1.VolumeAttributesClass)
if !ok {
framework.Failf("cast VolumeAttributesClass from %s", d.VolumeAttributesClass.FromFile)
}
}
gomega.Expect(vac).ToNot(gomega.BeNil(), "VolumeAttributesClass is unexpectantly nil")
return storageframework.CopyVolumeAttributesClass(vac, f.Namespace.Name, "e2e-vac")
}
func (d *driverDefinition) GetVolume(e2econfig *storageframework.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) {
if len(d.InlineVolumes) == 0 {
e2eskipper.Skipf("%s does not have any InlineVolumeAttributes defined", d.DriverInfo.Name)

2
test/e2e/storage/external/testdata/example.yaml vendored
View File

@@ -1,5 +1,7 @@
StorageClass:
FromExistingClassName: example
VolumeAttributesClass:
FromExistingClassName: example-vac
DriverInfo:
Name: example
RequiredAccessModes:

11
test/e2e/storage/framework/driver_operations.go
View File

@@ -21,6 +21,7 @@ import (
"fmt"
storagev1 "k8s.io/api/storage/v1"
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/storage/names"
"k8s.io/kubernetes/pkg/volume/util"
@@ -92,3 +93,13 @@ func GetStorageClass(
VolumeBindingMode: bindingMode,
}
}
// CopyVolumeAttributesClass constructs a new VolumeAttributesClass instance
// with a unique name that is based on namespace + suffix
// using the VolumeAttributesClass passed in as a parameter
func CopyVolumeAttributesClass(vac *storagev1alpha1.VolumeAttributesClass, ns string, suffix string) *storagev1alpha1.VolumeAttributesClass {
copy := vac.DeepCopy()
copy.ObjectMeta.Name = names.SimpleNameGenerator.GenerateName(ns + "-" + suffix)
copy.ResourceVersion = ""
return copy
}

10
test/e2e/storage/framework/testdriver.go
View File

@@ -22,6 +22,7 @@ import (
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/test/e2e/framework"
@@ -130,6 +131,15 @@ type SnapshottableTestDriver interface {
GetSnapshotClass(ctx context.Context, config *PerTestConfig, parameters map[string]string) *unstructured.Unstructured
}
// VolumeAttributesClassTestDriver represents an interface for a TestDriver that supports
// creating and modifying volumes via VolumeAttributesClass objects
type VolumeAttributesClassTestDriver interface {
TestDriver
// GetVolumeAttributesClass returns a VolumeAttributesClass to create/modify PVCs
// It will return nil if the TestDriver does not support VACs
GetVolumeAttributesClass(ctx context.Context, config *PerTestConfig) *storagev1alpha1.VolumeAttributesClass
}
// CustomTimeoutsTestDriver represents an interface fo a TestDriver that supports custom timeouts.
type CustomTimeoutsTestDriver interface {
TestDriver

26
test/e2e/storage/framework/volume_resource.go
View File

@@ -53,11 +53,19 @@ type VolumeResource struct {
// CreateVolumeResource constructs a VolumeResource for the current test. It knows how to deal with
// different test pattern volume types.
func CreateVolumeResource(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange) *VolumeResource {
return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes)
return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes, nil)
}
// CreateVolumeResource constructs a VolumeResource for the current test using the specified VAC name.
func CreateVolumeResourceWithVAC(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, vacName *string) *VolumeResource {
if pattern.VolType != DynamicPV {
framework.Failf("Creating volume with VAC only supported on dynamic PV tests")
}
return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes, vacName)
}
// CreateVolumeResourceWithAccessModes constructs a VolumeResource for the current test with the provided access modes.
func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, accessModes []v1.PersistentVolumeAccessMode) *VolumeResource {
func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, accessModes []v1.PersistentVolumeAccessMode, vacName *string) *VolumeResource {
r := VolumeResource{
Config: config,
Pattern: pattern,
@@ -107,7 +115,7 @@ func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver,
switch pattern.VolType {
case DynamicPV:
r.Pv, r.Pvc = createPVCPVFromDynamicProvisionSC(
ctx, f, dInfo.Name, claimSize, r.Sc, pattern.VolMode, accessModes)
ctx, f, dInfo.Name, claimSize, r.Sc, pattern.VolMode, accessModes, vacName)
r.VolSource = storageutils.CreateVolumeSource(r.Pvc.Name, false /* readOnly */)
case GenericEphemeralVolume:
driverVolumeSizeRange := dDriver.GetDriverInfo().SupportedSizeRange
@@ -287,17 +295,19 @@ func createPVCPVFromDynamicProvisionSC(
sc *storagev1.StorageClass,
volMode v1.PersistentVolumeMode,
accessModes []v1.PersistentVolumeAccessMode,
vacName *string,
) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) {
cs := f.ClientSet
ns := f.Namespace.Name
ginkgo.By("creating a claim")
pvcCfg := e2epv.PersistentVolumeClaimConfig{
NamePrefix: name,
ClaimSize: claimSize,
StorageClassName: &(sc.Name),
AccessModes: accessModes,
VolumeMode: &volMode,
NamePrefix: name,
ClaimSize: claimSize,
StorageClassName: &(sc.Name),
VolumeAttributesClassName: vacName,
AccessModes: accessModes,
VolumeMode: &volMode,
}
pvc := e2epv.MakePersistentVolumeClaim(pvcCfg, ns)

1
test/e2e/storage/testsuites/base.go
View File

@@ -82,6 +82,7 @@ var CSISuites = append(BaseSuites,
InitSnapshottableStressTestSuite,
InitVolumePerformanceTestSuite,
InitReadWriteOncePodTestSuite,
InitVolumeModifyTestSuite,
)
func getVolumeOpsFromMetricsForPlugin(ms testutil.Metrics, pluginName string) opCounts {

3
test/e2e/storage/testsuites/disruptive.go
View File

@@ -116,7 +116,8 @@ func (s *disruptiveTestSuite) DefineTests(driver storageframework.TestDriver, pa
l.config,
pattern,
testVolumeSizeRange,
accessModes)
accessModes,
nil)
}
}

4
test/e2e/storage/testsuites/readwriteoncepod.go
View File

@@ -133,7 +133,7 @@ func (t *readWriteOncePodTestSuite) DefineTests(driver storageframework.TestDriv
ginkgo.It("should preempt lower priority pods using ReadWriteOncePod volumes", func(ctx context.Context) {
// Create the ReadWriteOncePod PVC.
accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes)
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes, nil)
l.priorityClass = &schedulingv1.PriorityClass{
ObjectMeta: metav1.ObjectMeta{Name: "e2e-test-read-write-once-pod-" + string(uuid.NewUUID())},
@@ -189,7 +189,7 @@ func (t *readWriteOncePodTestSuite) DefineTests(driver storageframework.TestDriv
ginkgo.It("should block a second pod from using an in-use ReadWriteOncePod volume on the same node", func(ctx context.Context) {
// Create the ReadWriteOncePod PVC.
accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes)
l.volume = storageframework.CreateVolumeResourceWithAccessModes(ctx, driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes, nil)
podConfig := e2epod.Config{
NS: f.Namespace.Name,

294
test/e2e/storage/testsuites/volume_modify.go Normal file
View File

@@ -0,0 +1,294 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testsuites
import (
"context"
"encoding/json"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/features"
e2efeature "k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
admissionapi "k8s.io/pod-security-admission/api"
)
const (
modifyPollInterval = 2 * time.Second
setVACWaitPeriod = 30 * time.Second
modifyingConditionSyncWaitPeriod = 2 * time.Minute
modifyVolumeWaitPeriod = 10 * time.Minute
vacCleanupWaitPeriod = 30 * time.Second
)
type volumeModifyTestSuite struct {
tsInfo storageframework.TestSuiteInfo
}
// InitCustomVolumeModifyTestSuite returns volumeModifyTestSuite that implements TestSuite interface
// using custom test patterns
func InitCustomVolumeModifyTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
return &volumeModifyTestSuite{
tsInfo: storageframework.TestSuiteInfo{
Name: "volume-modify",
TestPatterns: patterns,
SupportedSizeRange: e2evolume.SizeRange{
Min: "1Gi",
},
TestTags: []interface{}{e2efeature.VolumeAttributesClass, framework.WithFeatureGate(features.VolumeAttributesClass)},
},
}
}
// InitVolumeModifyTestSuite returns volumeModifyTestSuite that implements TestSuite interface
// using testsuite default patterns
func InitVolumeModifyTestSuite() storageframework.TestSuite {
patterns := []storageframework.TestPattern{
storageframework.DefaultFsDynamicPV,
storageframework.BlockVolModeDynamicPV,
storageframework.NtfsDynamicPV,
}
return InitCustomVolumeModifyTestSuite(patterns)
}
func (v *volumeModifyTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
return v.tsInfo
}
func (v *volumeModifyTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
_, ok := driver.(storageframework.VolumeAttributesClassTestDriver)
if !ok {
e2eskipper.Skipf("Driver %q does not support VolumeAttributesClass tests - skipping", driver.GetDriverInfo().Name)
}
// Skip block storage tests if the driver we are testing against does not support block volumes
// TODO: This should be made generic so that it doesn't have to be re-written for every test that uses the BlockVolModeDynamicPV testcase
if !driver.GetDriverInfo().Capabilities[storageframework.CapBlock] && pattern.VolMode == v1.PersistentVolumeBlock {
e2eskipper.Skipf("Driver %q does not support block volume mode - skipping", driver.GetDriverInfo().Name)
}
}
func (v *volumeModifyTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
type local struct {
config *storageframework.PerTestConfig
resource *storageframework.VolumeResource
vac *storagev1alpha1.VolumeAttributesClass
}
var l local
// Beware that it also registers an AfterEach which renders f unusable. Any code using
// f must run inside an It or Context callback.
f := framework.NewFrameworkWithCustomTimeouts("volume-modify", storageframework.GetDriverTimeouts(driver))
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
init := func(ctx context.Context, createVolumeWithVAC bool) {
l = local{}
l.config = driver.PrepareTest(ctx, f)
vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver)
l.vac = vacDriver.GetVolumeAttributesClass(ctx, l.config)
if l.vac == nil {
e2eskipper.Skipf("Driver %q returned nil VolumeAttributesClass - skipping", driver.GetDriverInfo().Name)
}
ginkgo.By("Creating VolumeAttributesClass")
_, err := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Create(ctx, l.vac, metav1.CreateOptions{})
framework.ExpectNoError(err, "While creating VolumeAttributesClass")
ginkgo.By("Creating volume")
testVolumeSizeRange := v.GetTestSuiteInfo().SupportedSizeRange
if createVolumeWithVAC {
l.resource = storageframework.CreateVolumeResourceWithVAC(ctx, driver, l.config, pattern, testVolumeSizeRange, &l.vac.Name)
} else {
l.resource = storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, testVolumeSizeRange)
}
}
cleanup := func(ctx context.Context) {
var errs []error
if l.resource != nil {
ginkgo.By("Deleting VolumeResource")
errs = append(errs, l.resource.CleanupResource(ctx))
l.resource = nil
}
if l.vac != nil {
ginkgo.By("Deleting VAC")
CleanupVAC(ctx, l.vac, f.ClientSet, vacCleanupWaitPeriod)
l.vac = nil
}
framework.ExpectNoError(errors.NewAggregate(errs), "While cleaning up")
}
ginkgo.It("should create a volume with VAC", func(ctx context.Context) {
init(ctx, true /* volume created with VAC */)
ginkgo.DeferCleanup(cleanup)
ginkgo.By("Creating a pod with dynamically provisioned volume")
podConfig := e2epod.Config{
NS: f.Namespace.Name,
PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc},
SeLinuxLabel: e2epod.GetLinuxLabel(),
NodeSelection: l.config.ClientNodeSelection,
ImageID: e2epod.GetDefaultTestImageID(),
}
pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
framework.ExpectNoError(err, "While creating test pod with VAC")
createdPVC, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, l.resource.Pvc.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "While getting created PVC")
// Check VAC matches on created PVC, but not current VAC in status
gomega.Expect(vacMatches(createdPVC, l.vac.Name, false)).To(gomega.BeTrueBecause("Created PVC should match expected VAC"))
})
ginkgo.It("should modify volume with no VAC", func(ctx context.Context) {
init(ctx, false /* volume created without VAC */)
ginkgo.DeferCleanup(cleanup)
var err error
ginkgo.By("Creating a pod with dynamically provisioned volume")
podConfig := e2epod.Config{
NS: f.Namespace.Name,
PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc},
SeLinuxLabel: e2epod.GetLinuxLabel(),
NodeSelection: l.config.ClientNodeSelection,
ImageID: e2epod.GetDefaultTestImageID(),
}
pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
framework.ExpectNoError(err, "While creating pod for modifying")
ginkgo.By("Modifying PVC via VAC")
newPVC := SetPVCVACName(ctx, l.resource.Pvc, l.vac.Name, f.ClientSet, setVACWaitPeriod)
l.resource.Pvc = newPVC
gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil())
ginkgo.By("Waiting for modification to finish")
WaitForVolumeModification(ctx, l.resource.Pvc, f.ClientSet, modifyVolumeWaitPeriod)
pvcConditions := l.resource.Pvc.Status.Conditions
gomega.Expect(pvcConditions).To(gomega.BeEmpty(), "PVC should not have conditions")
})
ginkgo.It("should modify volume that already has a VAC", func(ctx context.Context) {
init(ctx, true /* volume created with VAC */)
ginkgo.DeferCleanup(cleanup)
vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver)
newVAC := vacDriver.GetVolumeAttributesClass(ctx, l.config)
gomega.Expect(newVAC).NotTo(gomega.BeNil())
_, err := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses().Create(ctx, newVAC, metav1.CreateOptions{})
framework.ExpectNoError(err, "While creating new VolumeAttributesClass")
ginkgo.DeferCleanup(CleanupVAC, newVAC, f.ClientSet, vacCleanupWaitPeriod)
ginkgo.By("Creating a pod with dynamically provisioned volume")
podConfig := e2epod.Config{
NS: f.Namespace.Name,
PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc},
SeLinuxLabel: e2epod.GetLinuxLabel(),
NodeSelection: l.config.ClientNodeSelection,
ImageID: e2epod.GetDefaultTestImageID(),
}
pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
framework.ExpectNoError(err, "While creating pod for modifying")
ginkgo.By("Modifying PVC via VAC")
newPVC := SetPVCVACName(ctx, l.resource.Pvc, newVAC.Name, f.ClientSet, setVACWaitPeriod)
l.resource.Pvc = newPVC
gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil())
ginkgo.By("Waiting for modification to finish")
WaitForVolumeModification(ctx, l.resource.Pvc, f.ClientSet, modifyVolumeWaitPeriod)
pvcConditions := l.resource.Pvc.Status.Conditions
gomega.Expect(pvcConditions).To(gomega.BeEmpty(), "PVC should not have conditions")
})
}
// SetPVCVACName sets the VolumeAttributesClassName on a PVC object
func SetPVCVACName(ctx context.Context, origPVC *v1.PersistentVolumeClaim, name string, c clientset.Interface, timeout time.Duration) *v1.PersistentVolumeClaim {
pvcName := origPVC.Name
var patchedPVC *v1.PersistentVolumeClaim
gomega.Eventually(ctx, func(g gomega.Gomega) {
var err error
patch := []map[string]interface{}{{"op": "replace", "path": "/spec/volumeAttributesClassName", "value": name}}
patchBytes, _ := json.Marshal(patch)
patchedPVC, err = c.CoreV1().PersistentVolumeClaims(origPVC.Namespace).Patch(ctx, pvcName, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
framework.ExpectNoError(err, "While patching PVC to add VAC name")
}, timeout, modifyPollInterval).Should(gomega.Succeed())
return patchedPVC
}
// WaitForVolumeModification waits for the volume to be modified
// The input PVC is assumed to have a VolumeAttributesClassName set
func WaitForVolumeModification(ctx context.Context, pvc *v1.PersistentVolumeClaim, c clientset.Interface, timeout time.Duration) {
pvName := pvc.Spec.VolumeName
gomega.Eventually(ctx, func(g gomega.Gomega) {
pv, err := c.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
framework.ExpectNoError(err, "While getting existing PV")
g.Expect(pv.Spec.VolumeAttributesClassName).NotTo(gomega.BeNil())
newPVC, err := c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "While getting new PVC")
g.Expect(vacMatches(newPVC, *pv.Spec.VolumeAttributesClassName, true)).To(gomega.BeTrueBecause("Modified PVC should match expected VAC"))
}, timeout, modifyPollInterval).Should(gomega.Succeed())
}
func CleanupVAC(ctx context.Context, vac *storagev1alpha1.VolumeAttributesClass, c clientset.Interface, timeout time.Duration) {
gomega.Eventually(ctx, func() error {
return c.StorageV1alpha1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})
}, timeout, modifyPollInterval).Should(gomega.BeNil())
}
func vacMatches(pvc *v1.PersistentVolumeClaim, expectedVac string, checkStatusCurrentVac bool) bool {
// Check the following to ensure the VAC matches and that all pending modifications are complete:
// 1. VAC Name matches Expected
// 2. PVC Modify Volume status is either nil or has an empty status string
// 3. PVC Status Current VAC Matches Expected (only if checkStatusCurrentVac is true)
// (3) is only expected to be true after a VAC is modified, but not when a VAC is used to create a volume
if pvc.Spec.VolumeAttributesClassName == nil || *pvc.Spec.VolumeAttributesClassName != expectedVac {
return false
}
if pvc.Status.ModifyVolumeStatus != nil && (pvc.Status.ModifyVolumeStatus.Status != "" || pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName != expectedVac) {
return false
}
if checkStatusCurrentVac {
if pvc.Status.CurrentVolumeAttributesClassName == nil || *pvc.Status.CurrentVolumeAttributesClassName != expectedVac {
return false
}
}
return true
}

25
test/e2e/storage/utils/create.go
View File

@@ -29,6 +29,7 @@ import (
v1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
storagev1 "k8s.io/api/storage/v1"
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -267,6 +268,7 @@ var factories = map[What]ItemFactory{
{"StatefulSet"}: &statefulSetFactory{},
{"Deployment"}: &deploymentFactory{},
{"StorageClass"}: &storageClassFactory{},
{"VolumeAttributesClass"}: &volumeAttributesClassFactory{},
{"CustomResourceDefinition"}: &customResourceDefinitionFactory{},
}
@@ -314,6 +316,8 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace,
PatchName(f, &item.Name)
case *storagev1.StorageClass:
PatchName(f, &item.Name)
case *storagev1alpha1.VolumeAttributesClass:
PatchName(f, &item.Name)
case *storagev1.CSIDriver:
PatchName(f, &item.Name)
case *v1.ServiceAccount:
@@ -618,6 +622,27 @@ func (*storageClassFactory) Create(ctx context.Context, f *framework.Framework,
}, nil
}
type volumeAttributesClassFactory struct{}
func (f *volumeAttributesClassFactory) New() runtime.Object {
return &storagev1alpha1.VolumeAttributesClass{}
}
func (*volumeAttributesClassFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) {
item, ok := i.(*storagev1alpha1.VolumeAttributesClass)
if !ok {
return nil, errorItemNotSupported
}
client := f.ClientSet.StorageV1alpha1().VolumeAttributesClasses()
if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil {
return nil, fmt.Errorf("create VolumeAttributesClass: %w", err)
}
return func(ctx context.Context) error {
return client.Delete(ctx, item.GetName(), metav1.DeleteOptions{})
}, nil
}
type csiDriverFactory struct{}
func (f *csiDriverFactory) New() runtime.Object {