366 lines
12 KiB
Go
366 lines
12 KiB
Go
/*
|
|
Copyright 2016 The Kubernetes Authors All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package reconciler
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"k8s.io/kubernetes/pkg/controller/volume/attacherdetacher"
|
|
"k8s.io/kubernetes/pkg/controller/volume/cache"
|
|
controllervolumetesting "k8s.io/kubernetes/pkg/controller/volume/testing"
|
|
"k8s.io/kubernetes/pkg/util/wait"
|
|
volumetesting "k8s.io/kubernetes/pkg/volume/testing"
|
|
)
|
|
|
|
const (
|
|
reconcilerLoopPeriod time.Duration = 0 * time.Millisecond
|
|
maxSafeToDetachDuration time.Duration = 50 * time.Millisecond
|
|
)
|
|
|
|
func Test_Run_Positive_DoNothing(t *testing.T) {
|
|
// Arrange
|
|
volumePluginMgr, fakePlugin := controllervolumetesting.GetTestVolumePluginMgr((t))
|
|
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
|
|
asw := cache.NewActualStateOfWorld(volumePluginMgr)
|
|
ad := attacherdetacher.NewAttacherDetacher(volumePluginMgr)
|
|
reconciler := NewReconciler(
|
|
reconcilerLoopPeriod, maxSafeToDetachDuration, dsw, asw, ad)
|
|
|
|
// Act
|
|
go reconciler.Run(wait.NeverStop)
|
|
|
|
// Assert
|
|
waitForNewAttacherCallCount(t, 0 /* expectedCallCount */, fakePlugin)
|
|
verifyNewAttacherCallCount(t, true /* expectZeroNewAttacherCallCount */, fakePlugin)
|
|
verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
|
|
waitForAttachCallCount(t, 0 /* expectedAttachCallCount */, fakePlugin)
|
|
waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
|
|
}
|
|
|
|
func Test_Run_Positive_OneDesiredVolumeAttach(t *testing.T) {
|
|
// Arrange
|
|
volumePluginMgr, fakePlugin := controllervolumetesting.GetTestVolumePluginMgr((t))
|
|
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
|
|
asw := cache.NewActualStateOfWorld(volumePluginMgr)
|
|
ad := attacherdetacher.NewAttacherDetacher(volumePluginMgr)
|
|
reconciler := NewReconciler(
|
|
reconcilerLoopPeriod, maxSafeToDetachDuration, dsw, asw, ad)
|
|
podName := "pod-name"
|
|
volumeName := "volume-name"
|
|
volumeSpec := controllervolumetesting.GetTestVolumeSpec(volumeName, volumeName)
|
|
nodeName := "node-name"
|
|
dsw.AddNode(nodeName)
|
|
volumeExists := dsw.VolumeExists(volumeName, nodeName)
|
|
if volumeExists {
|
|
t.Fatalf(
|
|
"Volume %q/node %q should not exist, but it does.",
|
|
volumeName,
|
|
nodeName)
|
|
}
|
|
|
|
_, podErr := dsw.AddPod(podName, volumeSpec, nodeName)
|
|
if podErr != nil {
|
|
t.Fatalf("AddPod failed. Expected: <no error> Actual: <%v>", podErr)
|
|
}
|
|
|
|
// Act
|
|
go reconciler.Run(wait.NeverStop)
|
|
|
|
// Assert
|
|
waitForNewAttacherCallCount(t, 1 /* expectedCallCount */, fakePlugin)
|
|
waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
|
|
verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
|
|
}
|
|
|
|
func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithMarkVolume(t *testing.T) {
|
|
// Arrange
|
|
volumePluginMgr, fakePlugin := controllervolumetesting.GetTestVolumePluginMgr((t))
|
|
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
|
|
asw := cache.NewActualStateOfWorld(volumePluginMgr)
|
|
ad := attacherdetacher.NewAttacherDetacher(volumePluginMgr)
|
|
reconciler := NewReconciler(
|
|
reconcilerLoopPeriod, maxSafeToDetachDuration, dsw, asw, ad)
|
|
podName := "pod-name"
|
|
volumeName := "volume-name"
|
|
volumeSpec := controllervolumetesting.GetTestVolumeSpec(volumeName, volumeName)
|
|
nodeName := "node-name"
|
|
dsw.AddNode(nodeName)
|
|
volumeExists := dsw.VolumeExists(volumeName, nodeName)
|
|
if volumeExists {
|
|
t.Fatalf(
|
|
"Volume %q/node %q should not exist, but it does.",
|
|
volumeName,
|
|
nodeName)
|
|
}
|
|
|
|
generatedVolumeName, podAddErr := dsw.AddPod(podName, volumeSpec, nodeName)
|
|
if podAddErr != nil {
|
|
t.Fatalf("AddPod failed. Expected: <no error> Actual: <%v>", podAddErr)
|
|
}
|
|
|
|
// Act
|
|
go reconciler.Run(wait.NeverStop)
|
|
|
|
// Assert
|
|
waitForNewAttacherCallCount(t, 1 /* expectedCallCount */, fakePlugin)
|
|
verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin)
|
|
waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
|
|
verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
|
|
waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
|
|
|
|
// Act
|
|
dsw.DeletePod(podName, generatedVolumeName, nodeName)
|
|
volumeExists = dsw.VolumeExists(generatedVolumeName, nodeName)
|
|
if volumeExists {
|
|
t.Fatalf(
|
|
"Deleted pod %q from volume %q/node %q. Volume should also be deleted but it still exists.",
|
|
podName,
|
|
generatedVolumeName,
|
|
nodeName)
|
|
}
|
|
asw.MarkVolumeNodeSafeToDetach(generatedVolumeName, nodeName)
|
|
|
|
// Assert -- Marked SafeToDetach
|
|
waitForNewDetacherCallCount(t, 1 /* expectedCallCount */, fakePlugin)
|
|
verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin)
|
|
waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
|
|
verifyNewDetacherCallCount(t, false /* expectZeroNewDetacherCallCount */, fakePlugin)
|
|
waitForDetachCallCount(t, 1 /* expectedDetachCallCount */, fakePlugin)
|
|
}
|
|
|
|
func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithoutMarkVolume(t *testing.T) {
|
|
// Arrange
|
|
volumePluginMgr, fakePlugin := controllervolumetesting.GetTestVolumePluginMgr((t))
|
|
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
|
|
asw := cache.NewActualStateOfWorld(volumePluginMgr)
|
|
ad := attacherdetacher.NewAttacherDetacher(volumePluginMgr)
|
|
reconciler := NewReconciler(
|
|
reconcilerLoopPeriod, maxSafeToDetachDuration, dsw, asw, ad)
|
|
podName := "pod-name"
|
|
volumeName := "volume-name"
|
|
volumeSpec := controllervolumetesting.GetTestVolumeSpec(volumeName, volumeName)
|
|
nodeName := "node-name"
|
|
dsw.AddNode(nodeName)
|
|
volumeExists := dsw.VolumeExists(volumeName, nodeName)
|
|
if volumeExists {
|
|
t.Fatalf(
|
|
"Volume %q/node %q should not exist, but it does.",
|
|
volumeName,
|
|
nodeName)
|
|
}
|
|
|
|
generatedVolumeName, podAddErr := dsw.AddPod(podName, volumeSpec, nodeName)
|
|
if podAddErr != nil {
|
|
t.Fatalf("AddPod failed. Expected: <no error> Actual: <%v>", podAddErr)
|
|
}
|
|
|
|
// Act
|
|
go reconciler.Run(wait.NeverStop)
|
|
|
|
// Assert
|
|
waitForNewAttacherCallCount(t, 1 /* expectedCallCount */, fakePlugin)
|
|
verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin)
|
|
waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
|
|
verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin)
|
|
waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin)
|
|
|
|
// Act
|
|
dsw.DeletePod(podName, generatedVolumeName, nodeName)
|
|
volumeExists = dsw.VolumeExists(generatedVolumeName, nodeName)
|
|
if volumeExists {
|
|
t.Fatalf(
|
|
"Deleted pod %q from volume %q/node %q. Volume should also be deleted but it still exists.",
|
|
podName,
|
|
generatedVolumeName,
|
|
nodeName)
|
|
}
|
|
|
|
// Assert -- Timer will triger detach
|
|
waitForNewDetacherCallCount(t, 1 /* expectedCallCount */, fakePlugin)
|
|
verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin)
|
|
waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin)
|
|
verifyNewDetacherCallCount(t, false /* expectZeroNewDetacherCallCount */, fakePlugin)
|
|
waitForDetachCallCount(t, 1 /* expectedDetachCallCount */, fakePlugin)
|
|
}
|
|
|
|
func waitForNewAttacherCallCount(
|
|
t *testing.T,
|
|
expectedCallCount int,
|
|
fakePlugin *volumetesting.FakeVolumePlugin) {
|
|
err := retryWithExponentialBackOff(
|
|
time.Duration(5*time.Millisecond),
|
|
func() (bool, error) {
|
|
actualCallCount := fakePlugin.GetNewAttacherCallCount()
|
|
if actualCallCount >= expectedCallCount {
|
|
return true, nil
|
|
}
|
|
t.Logf(
|
|
"Warning: Wrong NewAttacherCallCount. Expected: <%v> Actual: <%v>. Will retry.",
|
|
expectedCallCount,
|
|
actualCallCount)
|
|
return false, nil
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
t.Fatalf(
|
|
"Timed out waiting for NewAttacherCallCount. Expected: <%v> Actual: <%v>",
|
|
expectedCallCount,
|
|
fakePlugin.GetNewAttacherCallCount())
|
|
}
|
|
}
|
|
|
|
func waitForNewDetacherCallCount(
|
|
t *testing.T,
|
|
expectedCallCount int,
|
|
fakePlugin *volumetesting.FakeVolumePlugin) {
|
|
err := retryWithExponentialBackOff(
|
|
time.Duration(5*time.Millisecond),
|
|
func() (bool, error) {
|
|
actualCallCount := fakePlugin.GetNewDetacherCallCount()
|
|
if actualCallCount >= expectedCallCount {
|
|
return true, nil
|
|
}
|
|
t.Logf(
|
|
"Warning: Wrong NewDetacherCallCount. Expected: <%v> Actual: <%v>. Will retry.",
|
|
expectedCallCount,
|
|
actualCallCount)
|
|
return false, nil
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
t.Fatalf(
|
|
"Timed out waiting for NewDetacherCallCount. Expected: <%v> Actual: <%v>",
|
|
expectedCallCount,
|
|
fakePlugin.GetNewDetacherCallCount())
|
|
}
|
|
}
|
|
|
|
func waitForAttachCallCount(
|
|
t *testing.T,
|
|
expectedAttachCallCount int,
|
|
fakePlugin *volumetesting.FakeVolumePlugin) {
|
|
if len(fakePlugin.GetAttachers()) == 0 && expectedAttachCallCount == 0 {
|
|
return
|
|
}
|
|
|
|
err := retryWithExponentialBackOff(
|
|
time.Duration(5*time.Millisecond),
|
|
func() (bool, error) {
|
|
for i, attacher := range fakePlugin.GetAttachers() {
|
|
actualCallCount := attacher.GetAttachCallCount()
|
|
if actualCallCount == expectedAttachCallCount {
|
|
return true, nil
|
|
}
|
|
t.Logf(
|
|
"Warning: Wrong attacher[%v].GetAttachCallCount(). Expected: <%v> Actual: <%v>. Will try next attacher.",
|
|
i,
|
|
expectedAttachCallCount,
|
|
actualCallCount)
|
|
}
|
|
|
|
t.Logf(
|
|
"Warning: No attachers have expected AttachCallCount. Expected: <%v>. Will retry.",
|
|
expectedAttachCallCount)
|
|
return false, nil
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
t.Fatalf(
|
|
"No attachers have expected AttachCallCount. Expected: <%v>",
|
|
expectedAttachCallCount)
|
|
}
|
|
}
|
|
|
|
func waitForDetachCallCount(
|
|
t *testing.T,
|
|
expectedDetachCallCount int,
|
|
fakePlugin *volumetesting.FakeVolumePlugin) {
|
|
if len(fakePlugin.GetDetachers()) == 0 && expectedDetachCallCount == 0 {
|
|
return
|
|
}
|
|
|
|
err := retryWithExponentialBackOff(
|
|
time.Duration(5*time.Millisecond),
|
|
func() (bool, error) {
|
|
for i, detacher := range fakePlugin.GetDetachers() {
|
|
actualCallCount := detacher.GetDetachCallCount()
|
|
if actualCallCount == expectedDetachCallCount {
|
|
return true, nil
|
|
}
|
|
t.Logf(
|
|
"Wrong detacher[%v].GetDetachCallCount(). Expected: <%v> Actual: <%v>. Will try next detacher.",
|
|
i,
|
|
expectedDetachCallCount,
|
|
actualCallCount)
|
|
}
|
|
|
|
t.Logf(
|
|
"Warning: No detachers have expected DetachCallCount. Expected: <%v>. Will retry.",
|
|
expectedDetachCallCount)
|
|
return false, nil
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
t.Fatalf(
|
|
"No detachers have expected DetachCallCount. Expected: <%v>",
|
|
expectedDetachCallCount)
|
|
}
|
|
}
|
|
|
|
func verifyNewAttacherCallCount(
|
|
t *testing.T,
|
|
expectZeroNewAttacherCallCount bool,
|
|
fakePlugin *volumetesting.FakeVolumePlugin) {
|
|
|
|
if expectZeroNewAttacherCallCount &&
|
|
fakePlugin.GetNewAttacherCallCount() != 0 {
|
|
t.Fatalf(
|
|
"Wrong NewAttacherCallCount. Expected: <0> Actual: <%v>",
|
|
fakePlugin.GetNewAttacherCallCount())
|
|
}
|
|
}
|
|
|
|
func verifyNewDetacherCallCount(
|
|
t *testing.T,
|
|
expectZeroNewDetacherCallCount bool,
|
|
fakePlugin *volumetesting.FakeVolumePlugin) {
|
|
|
|
if expectZeroNewDetacherCallCount &&
|
|
fakePlugin.GetNewDetacherCallCount() != 0 {
|
|
t.Fatalf("Wrong NewDetacherCallCount. Expected: <0> Actual: <%v>",
|
|
fakePlugin.GetNewDetacherCallCount())
|
|
}
|
|
}
|
|
|
|
func retryWithExponentialBackOff(initialDuration time.Duration, fn wait.ConditionFunc) error {
|
|
backoff := wait.Backoff{
|
|
Duration: initialDuration,
|
|
Factor: 3,
|
|
Jitter: 0,
|
|
Steps: 6,
|
|
}
|
|
return wait.ExponentialBackoff(backoff, fn)
|
|
}
|
|
|
|
// t.Logf("asw: %v", asw.GetAttachedVolumes())
|
|
// t.Logf("dsw: %v", dsw.GetVolumesToAttach())
|