github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement external provisioning proposal

Browse Source 
In other words, add "provisioned-by" annotation to all PVCs
that should be provisioned dynamically.
This commit is contained in:
Jan Safranek
2016-11-02 14:13:34 +01:00
parent cc84673ebe
commit 18de83c641
4 changed files with 100 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/controller/volume/persistentvolume/framework_test.go
View File

@@ -750,6 +750,8 @@ func newClaim(name, claimUID, capacity, boundToVolume string, phase api.Persiste
switch a {
case storageutil.StorageClassAnnotation:
claim.Annotations[a] = "gold"
case annStorageProvisioner:
claim.Annotations[a] = mockPluginName
default:
claim.Annotations[a] = "yes"
}
@@ -786,6 +788,17 @@ func claimWithClass(className string, claims []*api.PersistentVolumeClaim) []*ap
return claims
}
// claimWithAnnotation saves given annotation into given claims.
// Meant to be used to compose claims specified inline in a test.
func claimWithAnnotation(name, value string, claims []*api.PersistentVolumeClaim) []*api.PersistentVolumeClaim {
if claims[0].Annotations == nil {
claims[0].Annotations = map[string]string{name: value}
} else {
claims[0].Annotations[name] = value
}
return claims
}
func testSyncClaim(ctrl *PersistentVolumeController, reactor *volumeReactor, test controllerTest) error {
return ctrl.syncClaim(test.initialClaims[0])
}

32
pkg/controller/volume/persistentvolume/provision_test.go
View File

@@ -79,7 +79,7 @@ var storageClasses = []*storage.StorageClass{
// call to storageClass 1, returning an error
var provision1Error = provisionCall{
ret: errors.New("Moc provisioner error"),
ret: errors.New("Mock provisioner error"),
expectedParameters: class1Parameters,
}
@@ -112,7 +112,7 @@ func TestProvisionSync(t *testing.T) {
newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
// Binding will be completed in the next syncClaim
newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
},
{
@@ -131,7 +131,7 @@ func TestProvisionSync(t *testing.T) {
novolumes,
novolumes,
newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
[]string{"Warning ProvisioningFailed"}, noerrors,
wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
},
@@ -141,7 +141,7 @@ func TestProvisionSync(t *testing.T) {
novolumes,
novolumes,
newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
[]string{"Warning ProvisioningFailed"}, noerrors,
wrapTestWithProvisionCalls([]provisionCall{provision1Error}, testSyncClaim),
},
@@ -165,7 +165,7 @@ func TestProvisionSync(t *testing.T) {
newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
// The claim would be bound in next syncClaim
newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
noevents, noerrors,
wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
// Create a volume before provisionClaimOperation starts.
@@ -184,7 +184,7 @@ func TestProvisionSync(t *testing.T) {
newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
// Binding will be completed in the next syncClaim
newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
noevents,
[]reactorError{
// Inject error to the first
@@ -201,7 +201,7 @@ func TestProvisionSync(t *testing.T) {
novolumes,
novolumes,
newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
[]string{"Warning ProvisioningFailed"},
[]reactorError{
// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -226,7 +226,7 @@ func TestProvisionSync(t *testing.T) {
novolumes,
novolumes,
newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
[]string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"},
[]reactorError{
// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -247,7 +247,7 @@ func TestProvisionSync(t *testing.T) {
novolumes,
novolumes,
newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
[]string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"},
[]reactorError{
// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -277,7 +277,7 @@ func TestProvisionSync(t *testing.T) {
novolumes,
novolumes,
newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
[]string{"Warning ProvisioningFailed"},
[]reactorError{
// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -305,7 +305,7 @@ func TestProvisionSync(t *testing.T) {
volumeWithClass("silver", newVolumeArray("pvc-uid11-13", "1Gi", "uid11-13", "claim11-13", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned)),
claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)),
// Binding will be completed in the next syncClaim
claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)),
claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending, annStorageProvisioner)),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision2Success}, testSyncClaim),
},
{
@@ -341,7 +341,8 @@ func TestProvisionSync(t *testing.T) {
novolumes,
novolumes,
claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)),
claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)),
claimWithAnnotation(annStorageProvisioner, "vendor.com/my-volume",
claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending))),
[]string{"Normal ExternalProvisioning"},
noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
},
@@ -368,7 +369,7 @@ func TestAlphaProvisionSync(t *testing.T) {
newVolumeArray("pvc-uid14-1", "1Gi", "uid14-1", "claim14-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
// Binding will be completed in the next syncClaim
newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
},
{
@@ -382,7 +383,7 @@ func TestAlphaProvisionSync(t *testing.T) {
},
newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
// Binding will be completed in the next syncClaim
newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
},
}
@@ -411,8 +412,7 @@ func TestProvisionMultiSync(t *testing.T) {
novolumes,
newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
newClaimArray("claim12-1", "uid12-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
// Binding will be completed in the next syncClaim
newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted),
newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted, annStorageProvisioner),
noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
},
}

63
pkg/controller/volume/persistentvolume/pv_controller.go
View File

@@ -117,6 +117,11 @@ const annBoundByController = "pv.kubernetes.io/bound-by-controller"
// recognize dynamically provisioned PVs in its decisions).
const annDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
// This annotation is added to a PVC that is supposed to be dynamically
// provisioned. Its value is name of volume plugin that is supposed to provision
// a volume for this PVC.
const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner"
// Name of a tag attached to a real volume in cloud (e.g. AWS EBS or GCE PD)
// with namespace of a persistent volume claim used to create this volume.
const cloudVolumeCreatedForClaimNamespaceTag = "kubernetes.io/created-for/pvc/namespace"
@@ -1197,6 +1202,42 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
claimClass := storageutil.GetClaimStorageClass(claim)
glog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass)
plugin, storageClass, err := ctrl.findProvisionablePlugin(claim)
if err != nil {
ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error())
glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err)
// The controller will retry provisioning the volume in every
// syncVolume() call.
return
}
if storageClass != nil {
// Add provisioner annotation so external provisioners know when to start
newClaim, err := ctrl.setClaimProvisioner(claim, storageClass)
if err != nil {
// Save failed, the controller will retry in the next sync
glog.V(2).Infof("error saving claim %s: %v", claimToClaimKey(claim), err)
return
}
claim = newClaim
}
if plugin == nil {
// findProvisionablePlugin returned no error nor plugin.
// This means that an unknown provisioner is requested. Report an event
// and wait for the external provisioner
if storageClass != nil {
msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner)
ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg)
glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg)
} else {
glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim))
}
return
}
// internal provisioning
// A previous doProvisionClaim may just have finished while we were waiting for
// the locks. Check that PV (with deterministic name) hasn't been provisioned
// yet.
@@ -1217,28 +1258,6 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
return
}
plugin, storageClass, err := ctrl.findProvisionablePlugin(claim)
if err != nil {
ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error())
glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err)
// The controller will retry provisioning the volume in every
// syncVolume() call.
return
}
if plugin == nil {
// findProvisionablePlugin returned no error nor plugin.
// This means that an unknown provisioner is requested. Report an event
// and wait for the external provisioner
if storageClass != nil {
msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner)
ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg)
glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg)
} else {
glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim))
}
return
}
// Gather provisioning options
tags := make(map[string]string)
tags[cloudVolumeCreatedForClaimNamespaceTag] = claim.Namespace

30
pkg/controller/volume/persistentvolume/pv_controller_base.go
View File

@@ -494,6 +494,36 @@ func (ctrl *PersistentVolumeController) upgradeVolumeFrom1_2(volume *api.Persist
return true
}
// setClaimProvisioner saves
// claim.Annotations[annStorageProvisioner] = class.Provisioner
func (ctrl *PersistentVolumeController) setClaimProvisioner(claim *api.PersistentVolumeClaim, class *storage.StorageClass) (*api.PersistentVolumeClaim, error) {
if val, ok := claim.Annotations[annDynamicallyProvisioned]; ok && val == class.Provisioner {
// annotation is already set, nothing to do
return claim, nil
}
// The volume from method args can be pointing to watcher cache. We must not
// modify these, therefore create a copy.
clone, err := conversion.NewCloner().DeepCopy(claim)
if err != nil {
return nil, fmt.Errorf("Error cloning pv: %v", err)
}
claimClone, ok := clone.(*api.PersistentVolumeClaim)
if !ok {
return nil, fmt.Errorf("Unexpected claim cast error : %v", claimClone)
}
api.SetMetaDataAnnotation(&claimClone.ObjectMeta, annStorageProvisioner, class.Provisioner)
newClaim, err := ctrl.kubeClient.Core().PersistentVolumeClaims(claim.Namespace).Update(claimClone)
if err != nil {
return newClaim, err
}
_, err = ctrl.storeClaimUpdate(newClaim)
if err != nil {
return newClaim, err
}
return newClaim, nil
}
// Stateless functions
func getClaimStatusForLogging(claim *api.PersistentVolumeClaim) string {