2472291790
PVC and containers shared the same ResourceRequirements struct to define their API. When resource claims were added, that struct got extended, which accidentally also changed the PVC API. To avoid such a mistake from happening again, PVC now uses its own VolumeResourceRequirements struct. The `Claims` field gets removed because risk of breaking someone is low: theoretically, YAML files which have a claims field for volumes now get rejected when validating against the OpenAPI. Such files have never made sense and should be fixed. Code that uses the struct definitions needs to be updated.
460 lines
16 KiB
Go
460 lines
16 KiB
Go
/*
|
|
Copyright 2019 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package operationexecutor
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
|
core "k8s.io/client-go/testing"
|
|
"k8s.io/client-go/tools/record"
|
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
|
"k8s.io/kubernetes/pkg/features"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/uuid"
|
|
fakeclient "k8s.io/client-go/kubernetes/fake"
|
|
"k8s.io/component-base/metrics/testutil"
|
|
"k8s.io/kubernetes/pkg/volume"
|
|
csitesting "k8s.io/kubernetes/pkg/volume/csi/testing"
|
|
volumetesting "k8s.io/kubernetes/pkg/volume/testing"
|
|
"k8s.io/kubernetes/pkg/volume/util"
|
|
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
|
|
)
|
|
|
|
// this method just tests the volume plugin name that's used in CompleteFunc, the same plugin is also used inside the
|
|
// generated func so there is no need to test the plugin name that's used inside generated function
|
|
func TestOperationGenerator_GenerateUnmapVolumeFunc_PluginName(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
pluginName string
|
|
pvSpec v1.PersistentVolumeSpec
|
|
probVolumePlugins []volume.VolumePlugin
|
|
}
|
|
|
|
testcases := []testcase{
|
|
{
|
|
name: "gce pd plugin: csi migration disabled",
|
|
pluginName: "fake-plugin",
|
|
pvSpec: v1.PersistentVolumeSpec{
|
|
PersistentVolumeSource: v1.PersistentVolumeSource{
|
|
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
|
|
}},
|
|
probVolumePlugins: volumetesting.ProbeVolumePlugins(volume.VolumeConfig{}),
|
|
},
|
|
}
|
|
|
|
for _, tc := range testcases {
|
|
expectedPluginName := tc.pluginName
|
|
volumePluginMgr, tmpDir := initTestPlugins(t, tc.probVolumePlugins, tc.pluginName)
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
operationGenerator := getTestOperationGenerator(volumePluginMgr)
|
|
|
|
pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID(string(uuid.NewUUID()))}}
|
|
volumeToUnmount := getTestVolumeToUnmount(pod, tc.pvSpec, tc.pluginName)
|
|
|
|
unmapVolumeFunc, e := operationGenerator.GenerateUnmapVolumeFunc(volumeToUnmount, nil)
|
|
if e != nil {
|
|
t.Fatalf("Error occurred while generating unmapVolumeFunc: %v", e)
|
|
}
|
|
|
|
m := util.StorageOperationMetric.WithLabelValues(expectedPluginName, "unmap_volume", "success", "false")
|
|
storageOperationDurationSecondsMetricBefore, _ := testutil.GetHistogramMetricCount(m)
|
|
|
|
var ee error
|
|
unmapVolumeFunc.CompleteFunc(volumetypes.CompleteFuncParam{Err: &ee})
|
|
|
|
storageOperationDurationSecondsMetricAfter, _ := testutil.GetHistogramMetricCount(m)
|
|
metricValueDiff := storageOperationDurationSecondsMetricAfter - storageOperationDurationSecondsMetricBefore
|
|
assert.Equal(t, uint64(1), metricValueDiff, tc.name)
|
|
}
|
|
}
|
|
|
|
func TestOperationGenerator_GenerateExpandAndRecoverVolumeFunc(t *testing.T) {
|
|
nodeResizePending := v1.PersistentVolumeClaimNodeResizePending
|
|
nodeResizeFailed := v1.PersistentVolumeClaimNodeResizeFailed
|
|
var tests = []struct {
|
|
name string
|
|
pvc *v1.PersistentVolumeClaim
|
|
pv *v1.PersistentVolume
|
|
recoverFeatureGate bool
|
|
disableNodeExpansion bool
|
|
// expectations of test
|
|
expectedResizeStatus v1.ClaimResourceStatus
|
|
expectedAllocatedSize resource.Quantity
|
|
expectResizeCall bool
|
|
}{
|
|
{
|
|
name: "pvc.spec.size > pv.spec.size, recover_expansion=on",
|
|
pvc: getTestPVC("test-vol0", "2G", "1G", "", nil),
|
|
pv: getTestPV("test-vol0", "1G"),
|
|
recoverFeatureGate: true,
|
|
expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
|
|
expectedAllocatedSize: resource.MustParse("2G"),
|
|
expectResizeCall: true,
|
|
},
|
|
{
|
|
name: "pvc.spec.size = pv.spec.size, recover_expansion=on",
|
|
pvc: getTestPVC("test-vol0", "1G", "1G", "", nil),
|
|
pv: getTestPV("test-vol0", "1G"),
|
|
recoverFeatureGate: true,
|
|
expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
|
|
expectedAllocatedSize: resource.MustParse("1G"),
|
|
expectResizeCall: true,
|
|
},
|
|
{
|
|
name: "pvc.spec.size = pv.spec.size, recover_expansion=on",
|
|
pvc: getTestPVC("test-vol0", "1G", "1G", "1G", &nodeResizePending),
|
|
pv: getTestPV("test-vol0", "1G"),
|
|
recoverFeatureGate: true,
|
|
expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
|
|
expectedAllocatedSize: resource.MustParse("1G"),
|
|
expectResizeCall: false,
|
|
},
|
|
{
|
|
name: "pvc.spec.size > pv.spec.size, recover_expansion=on, disable_node_expansion=true",
|
|
pvc: getTestPVC("test-vol0", "2G", "1G", "", nil),
|
|
pv: getTestPV("test-vol0", "1G"),
|
|
disableNodeExpansion: true,
|
|
recoverFeatureGate: true,
|
|
expectedResizeStatus: "",
|
|
expectedAllocatedSize: resource.MustParse("2G"),
|
|
expectResizeCall: true,
|
|
},
|
|
{
|
|
name: "pv.spec.size >= pvc.spec.size, recover_expansion=on, resize_status=node_expansion_failed",
|
|
pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizeFailed),
|
|
pv: getTestPV("test-vol0", "2G"),
|
|
recoverFeatureGate: true,
|
|
expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
|
|
expectedAllocatedSize: resource.MustParse("2G"),
|
|
expectResizeCall: false,
|
|
},
|
|
}
|
|
for i := range tests {
|
|
test := tests[i]
|
|
t.Run(test.name, func(t *testing.T) {
|
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, test.recoverFeatureGate)()
|
|
volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t)
|
|
fakePlugin.DisableNodeExpansion = test.disableNodeExpansion
|
|
pvc := test.pvc
|
|
pv := test.pv
|
|
og := getTestOperationGenerator(volumePluginMgr, pvc, pv)
|
|
rsOpts := inTreeResizeOpts{
|
|
pvc: pvc,
|
|
pv: pv,
|
|
resizerName: fakePlugin.GetPluginName(),
|
|
volumePlugin: fakePlugin,
|
|
}
|
|
ogInstance, _ := og.(*operationGenerator)
|
|
|
|
expansionResponse := ogInstance.expandAndRecoverFunction(rsOpts)
|
|
if expansionResponse.err != nil {
|
|
t.Fatalf("GenerateExpandAndRecoverVolumeFunc failed: %v", expansionResponse.err)
|
|
}
|
|
updatedPVC := expansionResponse.pvc
|
|
actualResizeStatus := updatedPVC.Status.AllocatedResourceStatuses[v1.ResourceStorage]
|
|
assert.Equal(t, actualResizeStatus, test.expectedResizeStatus)
|
|
actualAllocatedSize := updatedPVC.Status.AllocatedResources.Storage()
|
|
if test.expectedAllocatedSize.Cmp(*actualAllocatedSize) != 0 {
|
|
t.Fatalf("GenerateExpandAndRecoverVolumeFunc failed: expected allocated size %s, got %s", test.expectedAllocatedSize.String(), actualAllocatedSize.String())
|
|
}
|
|
if test.expectResizeCall != expansionResponse.resizeCalled {
|
|
t.Fatalf("GenerateExpandAndRecoverVolumeFunc failed: expected resize called %t, got %t", test.expectResizeCall, expansionResponse.resizeCalled)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOperationGenerator_nodeExpandVolume(t *testing.T) {
|
|
getSizeFunc := func(size string) *resource.Quantity {
|
|
x := resource.MustParse(size)
|
|
return &x
|
|
}
|
|
|
|
nodeResizeFailed := v1.PersistentVolumeClaimNodeResizeFailed
|
|
nodeResizePending := v1.PersistentVolumeClaimNodeResizePending
|
|
var tests = []struct {
|
|
name string
|
|
pvc *v1.PersistentVolumeClaim
|
|
pv *v1.PersistentVolume
|
|
|
|
// desired size, defaults to pv.Spec.Capacity
|
|
desiredSize *resource.Quantity
|
|
// actualSize, defaults to pvc.Status.Capacity
|
|
actualSize *resource.Quantity
|
|
|
|
// expectations of test
|
|
expectedResizeStatus v1.ClaimResourceStatus
|
|
expectedStatusSize resource.Quantity
|
|
resizeCallCount int
|
|
expectError bool
|
|
}{
|
|
{
|
|
name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_failed",
|
|
pvc: getTestPVC("test-vol0", "2G", "1G", "", &nodeResizeFailed),
|
|
pv: getTestPV("test-vol0", "2G"),
|
|
|
|
expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeFailed,
|
|
resizeCallCount: 0,
|
|
expectedStatusSize: resource.MustParse("1G"),
|
|
},
|
|
{
|
|
name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending",
|
|
pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizePending),
|
|
pv: getTestPV("test-vol0", "2G"),
|
|
expectedResizeStatus: "",
|
|
resizeCallCount: 1,
|
|
expectedStatusSize: resource.MustParse("2G"),
|
|
},
|
|
{
|
|
name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, reize_op=failing",
|
|
pvc: getTestPVC(volumetesting.AlwaysFailNodeExpansion, "2G", "1G", "2G", &nodeResizePending),
|
|
pv: getTestPV(volumetesting.AlwaysFailNodeExpansion, "2G"),
|
|
expectError: true,
|
|
expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeFailed,
|
|
resizeCallCount: 1,
|
|
expectedStatusSize: resource.MustParse("1G"),
|
|
},
|
|
{
|
|
name: "pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize = actualSize",
|
|
pvc: getTestPVC("test-vol0", "2G", "2G", "2G", nil),
|
|
pv: getTestPV("test-vol0", "2G"),
|
|
|
|
expectedResizeStatus: "",
|
|
resizeCallCount: 0,
|
|
expectedStatusSize: resource.MustParse("2G"),
|
|
},
|
|
{
|
|
name: "pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize",
|
|
pvc: getTestPVC("test-vol0", "2G", "2G", "2G", nil),
|
|
pv: getTestPV("test-vol0", "2G"),
|
|
desiredSize: getSizeFunc("2G"),
|
|
actualSize: getSizeFunc("1G"),
|
|
|
|
expectedResizeStatus: "",
|
|
resizeCallCount: 1,
|
|
expectedStatusSize: resource.MustParse("2G"),
|
|
},
|
|
{
|
|
name: "pv.spec.cap = pvc.status.cap, resizeStatus=node-expansion-failed, desiredSize > actualSize",
|
|
pvc: getTestPVC("test-vol0", "2G", "2G", "2G", &nodeResizeFailed),
|
|
pv: getTestPV("test-vol0", "2G"),
|
|
desiredSize: getSizeFunc("2G"),
|
|
actualSize: getSizeFunc("1G"),
|
|
|
|
expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeFailed,
|
|
resizeCallCount: 0,
|
|
expectedStatusSize: resource.MustParse("2G"),
|
|
},
|
|
}
|
|
for i := range tests {
|
|
test := tests[i]
|
|
t.Run(test.name, func(t *testing.T) {
|
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, true)()
|
|
volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t)
|
|
test.pv.Spec.ClaimRef = &v1.ObjectReference{
|
|
Namespace: test.pvc.Namespace,
|
|
Name: test.pvc.Name,
|
|
}
|
|
|
|
pvc := test.pvc
|
|
pv := test.pv
|
|
pod := getTestPod("test-pod", pvc.Name)
|
|
og := getTestOperatorGeneratorWithPVPVC(volumePluginMgr, pvc, pv)
|
|
vmt := VolumeToMount{
|
|
Pod: pod,
|
|
VolumeName: v1.UniqueVolumeName(pv.Name),
|
|
VolumeSpec: volume.NewSpecFromPersistentVolume(pv, false),
|
|
}
|
|
desiredSize := test.desiredSize
|
|
if desiredSize == nil {
|
|
desiredSize = pv.Spec.Capacity.Storage()
|
|
}
|
|
actualSize := test.actualSize
|
|
if actualSize == nil {
|
|
actualSize = pvc.Status.Capacity.Storage()
|
|
}
|
|
pluginResizeOpts := volume.NodeResizeOptions{
|
|
VolumeSpec: vmt.VolumeSpec,
|
|
NewSize: *desiredSize,
|
|
OldSize: *actualSize,
|
|
}
|
|
|
|
ogInstance, _ := og.(*operationGenerator)
|
|
_, err := ogInstance.nodeExpandVolume(vmt, nil, pluginResizeOpts)
|
|
|
|
if !test.expectError && err != nil {
|
|
t.Errorf("For test %s, expected no error got: %v", test.name, err)
|
|
}
|
|
if test.expectError && err == nil {
|
|
t.Errorf("For test %s, expected error but got none", test.name)
|
|
}
|
|
if test.resizeCallCount != fakePlugin.NodeExpandCallCount {
|
|
t.Errorf("for test %s, expected node-expand call count to be %d, got %d", test.name, test.resizeCallCount, fakePlugin.NodeExpandCallCount)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func getTestPod(podName, pvcName string) *v1.Pod {
|
|
return &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
UID: "test-pod-uid",
|
|
Namespace: "ns",
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: pvcName,
|
|
VolumeSource: v1.VolumeSource{
|
|
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
|
ClaimName: pvcName,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func getTestPVC(volumeName string, specSize, statusSize, allocatedSize string, resizeStatus *v1.ClaimResourceStatus) *v1.PersistentVolumeClaim {
|
|
pvc := &v1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "claim01",
|
|
Namespace: "ns",
|
|
UID: "test-uid",
|
|
},
|
|
Spec: v1.PersistentVolumeClaimSpec{
|
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
|
Resources: v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse(specSize)}},
|
|
VolumeName: volumeName,
|
|
},
|
|
Status: v1.PersistentVolumeClaimStatus{
|
|
Phase: v1.ClaimBound,
|
|
},
|
|
}
|
|
if len(statusSize) > 0 {
|
|
pvc.Status.Capacity = v1.ResourceList{v1.ResourceStorage: resource.MustParse(statusSize)}
|
|
}
|
|
if len(allocatedSize) > 0 {
|
|
pvc.Status.AllocatedResources = v1.ResourceList{v1.ResourceStorage: resource.MustParse(allocatedSize)}
|
|
}
|
|
if resizeStatus != nil {
|
|
pvc.Status.AllocatedResourceStatuses = map[v1.ResourceName]v1.ClaimResourceStatus{
|
|
v1.ResourceStorage: *resizeStatus,
|
|
}
|
|
}
|
|
return pvc
|
|
}
|
|
|
|
func getTestPV(volumeName string, specSize string) *v1.PersistentVolume {
|
|
return &v1.PersistentVolume{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: volumeName,
|
|
UID: "test-uid",
|
|
},
|
|
Spec: v1.PersistentVolumeSpec{
|
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
|
Capacity: v1.ResourceList{
|
|
v1.ResourceStorage: resource.MustParse(specSize),
|
|
},
|
|
},
|
|
Status: v1.PersistentVolumeStatus{
|
|
Phase: v1.VolumeBound,
|
|
},
|
|
}
|
|
}
|
|
|
|
func getTestOperationGenerator(volumePluginMgr *volume.VolumePluginMgr, objects ...runtime.Object) OperationGenerator {
|
|
fakeKubeClient := fakeclient.NewSimpleClientset(objects...)
|
|
fakeRecorder := &record.FakeRecorder{}
|
|
fakeHandler := volumetesting.NewBlockVolumePathHandler()
|
|
operationGenerator := NewOperationGenerator(
|
|
fakeKubeClient,
|
|
volumePluginMgr,
|
|
fakeRecorder,
|
|
fakeHandler)
|
|
return operationGenerator
|
|
}
|
|
|
|
func getTestOperatorGeneratorWithPVPVC(volumePluginMgr *volume.VolumePluginMgr, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) OperationGenerator {
|
|
fakeKubeClient := fakeclient.NewSimpleClientset(pvc, pv)
|
|
fakeKubeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, pvc, nil
|
|
})
|
|
fakeKubeClient.AddReactor("get", "persistentvolumes", func(action core.Action) (bool, runtime.Object, error) {
|
|
return true, pv, nil
|
|
})
|
|
fakeKubeClient.AddReactor("patch", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
|
|
if action.GetSubresource() == "status" {
|
|
return true, pvc, nil
|
|
}
|
|
return true, nil, fmt.Errorf("no reaction implemented for %s", action)
|
|
})
|
|
|
|
fakeRecorder := &record.FakeRecorder{}
|
|
fakeHandler := volumetesting.NewBlockVolumePathHandler()
|
|
operationGenerator := NewOperationGenerator(
|
|
fakeKubeClient,
|
|
volumePluginMgr,
|
|
fakeRecorder,
|
|
fakeHandler)
|
|
return operationGenerator
|
|
}
|
|
|
|
func getTestVolumeToUnmount(pod *v1.Pod, pvSpec v1.PersistentVolumeSpec, pluginName string) MountedVolume {
|
|
volumeSpec := &volume.Spec{
|
|
PersistentVolume: &v1.PersistentVolume{
|
|
Spec: pvSpec,
|
|
},
|
|
}
|
|
volumeToUnmount := MountedVolume{
|
|
VolumeName: v1.UniqueVolumeName("pd-volume"),
|
|
PodUID: pod.UID,
|
|
PluginName: pluginName,
|
|
VolumeSpec: volumeSpec,
|
|
}
|
|
return volumeToUnmount
|
|
}
|
|
|
|
func initTestPlugins(t *testing.T, plugs []volume.VolumePlugin, pluginName string) (*volume.VolumePluginMgr, string) {
|
|
client := fakeclient.NewSimpleClientset()
|
|
pluginMgr, _, tmpDir := csitesting.NewTestPlugin(t, client)
|
|
|
|
err := pluginMgr.InitPlugins(plugs, nil, pluginMgr.Host)
|
|
if err != nil {
|
|
t.Fatalf("Can't init volume plugins: %v", err)
|
|
}
|
|
|
|
_, e := pluginMgr.FindPluginByName(pluginName)
|
|
if e != nil {
|
|
t.Fatalf("Can't find the plugin by name: %s", pluginName)
|
|
}
|
|
|
|
return pluginMgr, tmpDir
|
|
}
|