github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed race condition in pv binder

Browse Source
This commit is contained in:
markturansky
2015-10-28 11:03:04 -04:00
parent eb3de9c441
commit b9b8cf7fff
5 changed files with 341 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
pkg/controller/persistentvolume/persistentvolume_claim_binder_controller_test.go
View File

@@ -51,6 +51,97 @@ func TestRunStop(t *testing.T) {
}
}
func TestClaimRace(t *testing.T) {
c1 := &api.PersistentVolumeClaim{
ObjectMeta: api.ObjectMeta{
Name: "c1",
},
Spec: api.PersistentVolumeClaimSpec{
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("3Gi"),
},
},
},
Status: api.PersistentVolumeClaimStatus{
Phase: api.ClaimPending,
},
}
c1.ObjectMeta.SelfLink = testapi.Default.SelfLink("pvc", "")
c2 := &api.PersistentVolumeClaim{
ObjectMeta: api.ObjectMeta{
Name: "c2",
},
Spec: api.PersistentVolumeClaimSpec{
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("3Gi"),
},
},
},
Status: api.PersistentVolumeClaimStatus{
Phase: api.ClaimPending,
},
}
c2.ObjectMeta.SelfLink = testapi.Default.SelfLink("pvc", "")
v := &api.PersistentVolume{
ObjectMeta: api.ObjectMeta{
Name: "foo",
},
Spec: api.PersistentVolumeSpec{
AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce},
Capacity: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10Gi"),
},
PersistentVolumeSource: api.PersistentVolumeSource{
HostPath: &api.HostPathVolumeSource{
Path: "/tmp/data01",
},
},
},
Status: api.PersistentVolumeStatus{
Phase: api.VolumePending,
},
}
volumeIndex := NewPersistentVolumeOrderedIndex()
mockClient := &mockBinderClient{}
plugMgr := volume.VolumePluginMgr{}
plugMgr.InitPlugins(host_path.ProbeRecyclableVolumePlugins(newMockRecycler, volume.VolumeConfig{}), volume.NewFakeVolumeHost("/tmp/fake", nil, nil))
// adds the volume to the index, making the volume available
syncVolume(volumeIndex, mockClient, v)
if mockClient.volume.Status.Phase != api.VolumeAvailable {
t.Errorf("Expected phase %s but got %s", api.VolumeAvailable, mockClient.volume.Status.Phase)
}
if _, exists, _ := volumeIndex.Get(v); !exists {
t.Errorf("Expected to find volume in index but it did not exist")
}
// an initial sync for a claim matches the volume
err := syncClaim(volumeIndex, mockClient, c1)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if c1.Status.Phase != api.ClaimBound {
t.Errorf("Expected phase %s but got %s", api.ClaimBound, c1.Status.Phase)
}
// before the volume gets updated w/ claimRef, a 2nd claim can attempt to bind and find the same volume
err = syncClaim(volumeIndex, mockClient, c2)
if err != nil {
t.Errorf("unexpected error for unmatched claim: %v", err)
}
if c2.Status.Phase != api.ClaimPending {
t.Errorf("Expected phase %s but got %s", api.ClaimPending, c2.Status.Phase)
}
}
func TestExampleObjects(t *testing.T) {
scenarios := map[string]struct {
expected interface{}
@@ -188,6 +279,11 @@ func TestBindingWithExamples(t *testing.T) {
}
pv.ObjectMeta.SelfLink = testapi.Default.SelfLink("pv", "")
// the default value of the PV is Pending. if processed at least once, its status in etcd is Available.
// There was a bug where only Pending volumes were being indexed and made ready for claims.
// Test that !Pending gets correctly added
pv.Status.Phase = api.VolumeAvailable
claim, error := client.PersistentVolumeClaims("ns").Get("any")
if error != nil {
t.Errorf("Unexpected error getting PVC from client: %v", err)
@@ -211,145 +307,65 @@ func TestBindingWithExamples(t *testing.T) {
// adds the volume to the index, making the volume available
syncVolume(volumeIndex, mockClient, pv)
if pv.Status.Phase != api.VolumeAvailable {
t.Errorf("Expected phase %s but got %s", api.VolumeBound, pv.Status.Phase)
if mockClient.volume.Status.Phase != api.VolumeAvailable {
t.Errorf("Expected phase %s but got %s", api.VolumeAvailable, mockClient.volume.Status.Phase)
}
// an initial sync for a claim will bind it to an unbound volume, triggers state change
// an initial sync for a claim will bind it to an unbound volume
syncClaim(volumeIndex, mockClient, claim)
// state change causes another syncClaim to update statuses
syncClaim(volumeIndex, mockClient, claim)
// claim updated volume's status, causing an update and syncVolume call
syncVolume(volumeIndex, mockClient, pv)
if pv.Spec.ClaimRef == nil {
t.Errorf("Expected ClaimRef but got nil for pv.Status.ClaimRef: %+v\n", pv)
// bind expected on pv.Spec but status update hasn't happened yet
if mockClient.volume.Spec.ClaimRef == nil {
t.Errorf("Expected ClaimRef but got nil for pv.Status.ClaimRef\n")
}
if pv.Status.Phase != api.VolumeBound {
t.Errorf("Expected phase %s but got %s", api.VolumeBound, pv.Status.Phase)
if mockClient.volume.Status.Phase != api.VolumeAvailable {
t.Errorf("Expected phase %s but got %s", api.VolumeAvailable, mockClient.volume.Status.Phase)
}
if claim.Status.Phase != api.ClaimBound {
if mockClient.claim.Spec.VolumeName != pv.Name {
t.Errorf("Expected claim.Spec.VolumeName %s but got %s", mockClient.claim.Spec.VolumeName, pv.Name)
}
if mockClient.claim.Status.Phase != api.ClaimBound {
t.Errorf("Expected phase %s but got %s", api.ClaimBound, claim.Status.Phase)
}
if len(claim.Status.AccessModes) != len(pv.Spec.AccessModes) {
t.Errorf("Expected phase %s but got %s", api.ClaimBound, claim.Status.Phase)
}
if claim.Status.AccessModes[0] != pv.Spec.AccessModes[0] {
t.Errorf("Expected access mode %s but got %s", claim.Status.AccessModes[0], pv.Spec.AccessModes[0])
// state changes in pvc triggers sync that sets pv attributes to pvc.Status
syncClaim(volumeIndex, mockClient, claim)
if len(mockClient.claim.Status.AccessModes) == 0 {
t.Errorf("Expected %d access modes but got 0", len(pv.Spec.AccessModes))
}
// pretend the user deleted their claim
// persisting the bind to pv.Spec.ClaimRef triggers a sync
syncVolume(volumeIndex, mockClient, mockClient.volume)
if mockClient.volume.Status.Phase != api.VolumeBound {
t.Errorf("Expected phase %s but got %s", api.VolumeBound, mockClient.volume.Status.Phase)
}
// pretend the user deleted their claim. periodic resync picks it up.
mockClient.claim = nil
syncVolume(volumeIndex, mockClient, pv)
syncVolume(volumeIndex, mockClient, mockClient.volume)
if pv.Status.Phase != api.VolumeReleased {
t.Errorf("Expected phase %s but got %s", api.VolumeReleased, pv.Status.Phase)
if mockClient.volume.Status.Phase != api.VolumeReleased {
t.Errorf("Expected phase %s but got %s", api.VolumeReleased, mockClient.volume.Status.Phase)
}
if pv.Spec.ClaimRef == nil {
t.Errorf("Expected non-nil ClaimRef: %+v", pv.Spec)
}
mockClient.volume = pv
// released volumes with a PersistentVolumeReclaimPolicy (recycle/delete) can have further processing
err = recycler.reclaimVolume(pv)
err = recycler.reclaimVolume(mockClient.volume)
if err != nil {
t.Errorf("Unexpected error reclaiming volume: %+v", err)
}
if pv.Status.Phase != api.VolumePending {
t.Errorf("Expected phase %s but got %s", api.VolumePending, pv.Status.Phase)
if mockClient.volume.Status.Phase != api.VolumePending {
t.Errorf("Expected phase %s but got %s", api.VolumePending, mockClient.volume.Status.Phase)
}
// after the recycling changes the phase to Pending, the binder picks up again
// to remove any vestiges of binding and make the volume Available again
syncVolume(volumeIndex, mockClient, pv)
syncVolume(volumeIndex, mockClient, mockClient.volume)
if pv.Status.Phase != api.VolumeAvailable {
t.Errorf("Expected phase %s but got %s", api.VolumeAvailable, pv.Status.Phase)
if mockClient.volume.Status.Phase != api.VolumeAvailable {
t.Errorf("Expected phase %s but got %s", api.VolumeAvailable, mockClient.volume.Status.Phase)
}
if pv.Spec.ClaimRef != nil {
t.Errorf("Expected nil ClaimRef: %+v", pv.Spec)
}
}
func TestMissingFromIndex(t *testing.T) {
o := testclient.NewObjects(api.Scheme, api.Scheme)
if err := testclient.AddObjectsFromPath("../../../docs/user-guide/persistent-volumes/claims/claim-01.yaml", o, api.Scheme); err != nil {
t.Fatal(err)
}
if err := testclient.AddObjectsFromPath("../../../docs/user-guide/persistent-volumes/volumes/local-01.yaml", o, api.Scheme); err != nil {
t.Fatal(err)
}
client := &testclient.Fake{}
client.AddReactor("*", "*", testclient.ObjectReaction(o, api.RESTMapper))
pv, err := client.PersistentVolumes().Get("any")
if err != nil {
t.Errorf("Unexpected error getting PV from client: %v", err)
}
pv.ObjectMeta.SelfLink = testapi.Default.SelfLink("pv", "")
claim, error := client.PersistentVolumeClaims("ns").Get("any")
if error != nil {
t.Errorf("Unexpected error getting PVC from client: %v", err)
}
claim.ObjectMeta.SelfLink = testapi.Default.SelfLink("pvc", "")
volumeIndex := NewPersistentVolumeOrderedIndex()
mockClient := &mockBinderClient{
volume: pv,
claim: claim,
}
// the default value of the PV is Pending.
// if has previously been processed by the binder, it's status in etcd would be Available.
// Only Pending volumes were being indexed and made ready for claims.
pv.Status.Phase = api.VolumeAvailable
// adds the volume to the index, making the volume available
syncVolume(volumeIndex, mockClient, pv)
if pv.Status.Phase != api.VolumeAvailable {
t.Errorf("Expected phase %s but got %s", api.VolumeBound, pv.Status.Phase)
}
// an initial sync for a claim will bind it to an unbound volume, triggers state change
err = syncClaim(volumeIndex, mockClient, claim)
if err != nil {
t.Fatalf("Expected Clam to be bound, instead got an error: %+v\n", err)
}
// state change causes another syncClaim to update statuses
syncClaim(volumeIndex, mockClient, claim)
// claim updated volume's status, causing an update and syncVolume call
syncVolume(volumeIndex, mockClient, pv)
if pv.Spec.ClaimRef == nil {
t.Errorf("Expected ClaimRef but got nil for pv.Status.ClaimRef: %+v\n", pv)
}
if pv.Status.Phase != api.VolumeBound {
t.Errorf("Expected phase %s but got %s", api.VolumeBound, pv.Status.Phase)
}
if claim.Status.Phase != api.ClaimBound {
t.Errorf("Expected phase %s but got %s", api.ClaimBound, claim.Status.Phase)
}
if len(claim.Status.AccessModes) != len(pv.Spec.AccessModes) {
t.Errorf("Expected phase %s but got %s", api.ClaimBound, claim.Status.Phase)
}
if claim.Status.AccessModes[0] != pv.Spec.AccessModes[0] {
t.Errorf("Expected access mode %s but got %s", claim.Status.AccessModes[0], pv.Spec.AccessModes[0])
}
// pretend the user deleted their claim
mockClient.claim = nil
syncVolume(volumeIndex, mockClient, pv)
if pv.Status.Phase != api.VolumeReleased {
t.Errorf("Expected phase %s but got %s", api.VolumeReleased, pv.Status.Phase)
if mockClient.volume.Spec.ClaimRef != nil {
t.Errorf("Expected nil ClaimRef: %+v", mockClient.volume.Spec.ClaimRef)
}
}
@@ -363,7 +379,8 @@ func (c *mockBinderClient) GetPersistentVolume(name string) (*api.PersistentVolu
}
func (c *mockBinderClient) UpdatePersistentVolume(volume *api.PersistentVolume) (*api.PersistentVolume, error) {
return volume, nil
c.volume = volume
return c.volume, nil
}
func (c *mockBinderClient) DeletePersistentVolume(volume *api.PersistentVolume) error {
@@ -372,7 +389,8 @@ func (c *mockBinderClient) DeletePersistentVolume(volume *api.PersistentVolume)
}
func (c *mockBinderClient) UpdatePersistentVolumeStatus(volume *api.PersistentVolume) (*api.PersistentVolume, error) {
return volume, nil
c.volume = volume
return c.volume, nil
}
func (c *mockBinderClient) GetPersistentVolumeClaim(namespace, name string) (*api.PersistentVolumeClaim, error) {
@@ -384,11 +402,13 @@ func (c *mockBinderClient) GetPersistentVolumeClaim(namespace, name string) (*ap
}
func (c *mockBinderClient) UpdatePersistentVolumeClaim(claim *api.PersistentVolumeClaim) (*api.PersistentVolumeClaim, error) {
return claim, nil
c.claim = claim
return c.claim, nil
}
func (c *mockBinderClient) UpdatePersistentVolumeClaimStatus(claim *api.PersistentVolumeClaim) (*api.PersistentVolumeClaim, error) {
return claim, nil
c.claim = claim
return c.claim, nil
}
func newMockRecycler(spec *volume.Spec, host volume.VolumeHost, config volume.VolumeConfig) (volume.Recycler, error) {