Implement external provisioning proposal

In other words, add "provisioned-by" annotation to all PVCs
that should be provisioned dynamically.

pkg/controller/volume/persistentvolume/framework_test.go

@@ -750,6 +750,8 @@ func newClaim(name, claimUID, capacity, boundToVolume string, phase api.Persiste
 			switch a {
 			case storageutil.StorageClassAnnotation:
 				claim.Annotations[a] = "gold"
+			case annStorageProvisioner:
+				claim.Annotations[a] = mockPluginName
 			default:
 				claim.Annotations[a] = "yes"
 			}
@@ -786,6 +788,17 @@ func claimWithClass(className string, claims []*api.PersistentVolumeClaim) []*ap
 	return claims
 }
 
+// claimWithAnnotation saves given annotation into given claims.
+// Meant to be used to compose claims specified inline in a test.
+func claimWithAnnotation(name, value string, claims []*api.PersistentVolumeClaim) []*api.PersistentVolumeClaim {
+	if claims[0].Annotations == nil {
+		claims[0].Annotations = map[string]string{name: value}
+	} else {
+		claims[0].Annotations[name] = value
+	}
+	return claims
+}
+
 func testSyncClaim(ctrl *PersistentVolumeController, reactor *volumeReactor, test controllerTest) error {
 	return ctrl.syncClaim(test.initialClaims[0])
 }

pkg/controller/volume/persistentvolume/provision_test.go

@@ -79,7 +79,7 @@ var storageClasses = []*storage.StorageClass{
 
 // call to storageClass 1, returning an error
 var provision1Error = provisionCall{
-	ret:                errors.New("Moc provisioner error"),
+	ret:                errors.New("Mock provisioner error"),
 	expectedParameters: class1Parameters,
 }
 
@@ -112,7 +112,7 @@ func TestProvisionSync(t *testing.T) {
 			newVolumeArray("pvc-uid11-1", "1Gi", "uid11-1", "claim11-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
 			newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
 			// Binding will be completed in the next syncClaim
-			newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-1", "uid11-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
 		},
 		{
@@ -131,7 +131,7 @@ func TestProvisionSync(t *testing.T) {
 			novolumes,
 			novolumes,
 			newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
-			newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-3", "uid11-3", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			[]string{"Warning ProvisioningFailed"}, noerrors,
 			wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
 		},
@@ -141,7 +141,7 @@ func TestProvisionSync(t *testing.T) {
 			novolumes,
 			novolumes,
 			newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
-			newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-4", "uid11-4", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			[]string{"Warning ProvisioningFailed"}, noerrors,
 			wrapTestWithProvisionCalls([]provisionCall{provision1Error}, testSyncClaim),
 		},
@@ -165,7 +165,7 @@ func TestProvisionSync(t *testing.T) {
 			newVolumeArray("pvc-uid11-7", "1Gi", "uid11-7", "claim11-7", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
 			newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
 			// The claim would be bound in next syncClaim
-			newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-7", "uid11-7", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			noevents, noerrors,
 			wrapTestWithInjectedOperation(wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), func(ctrl *PersistentVolumeController, reactor *volumeReactor) {
 				// Create a volume before provisionClaimOperation starts.
@@ -184,7 +184,7 @@ func TestProvisionSync(t *testing.T) {
 			newVolumeArray("pvc-uid11-8", "1Gi", "uid11-8", "claim11-8", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
 			newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
 			// Binding will be completed in the next syncClaim
-			newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-8", "uid11-8", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			noevents,
 			[]reactorError{
 				// Inject error to the first
@@ -201,7 +201,7 @@ func TestProvisionSync(t *testing.T) {
 			novolumes,
 			novolumes,
 			newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
-			newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-9", "uid11-9", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			[]string{"Warning ProvisioningFailed"},
 			[]reactorError{
 				// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -226,7 +226,7 @@ func TestProvisionSync(t *testing.T) {
 			novolumes,
 			novolumes,
 			newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
-			newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-10", "uid11-10", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			[]string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"},
 			[]reactorError{
 				// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -247,7 +247,7 @@ func TestProvisionSync(t *testing.T) {
 			novolumes,
 			novolumes,
 			newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
-			newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-11", "uid11-11", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			[]string{"Warning ProvisioningFailed", "Warning ProvisioningCleanupFailed"},
 			[]reactorError{
 				// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -277,7 +277,7 @@ func TestProvisionSync(t *testing.T) {
 			novolumes,
 			novolumes,
 			newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
-			newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
+			newClaimArray("claim11-12", "uid11-12", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation, annStorageProvisioner),
 			[]string{"Warning ProvisioningFailed"},
 			[]reactorError{
 				// Inject error to five kubeclient.PersistentVolumes.Create()
@@ -305,7 +305,7 @@ func TestProvisionSync(t *testing.T) {
 			volumeWithClass("silver", newVolumeArray("pvc-uid11-13", "1Gi", "uid11-13", "claim11-13", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned)),
 			claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)),
 			// Binding will be completed in the next syncClaim
-			claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending)),
+			claimWithClass("silver", newClaimArray("claim11-13", "uid11-13", "1Gi", "", api.ClaimPending, annStorageProvisioner)),
 			noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision2Success}, testSyncClaim),
 		},
 		{
@@ -341,7 +341,8 @@ func TestProvisionSync(t *testing.T) {
 			novolumes,
 			novolumes,
 			claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)),
-			claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending)),
+			claimWithAnnotation(annStorageProvisioner, "vendor.com/my-volume",
+				claimWithClass("external", newClaimArray("claim11-17", "uid11-17", "1Gi", "", api.ClaimPending))),
 			[]string{"Normal ExternalProvisioning"},
 			noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim),
 		},
@@ -368,7 +369,7 @@ func TestAlphaProvisionSync(t *testing.T) {
 			newVolumeArray("pvc-uid14-1", "1Gi", "uid14-1", "claim14-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
 			newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
 			// Binding will be completed in the next syncClaim
-			newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
+			newClaimArray("claim14-1", "uid14-1", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
 			noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
 		},
 		{
@@ -382,7 +383,7 @@ func TestAlphaProvisionSync(t *testing.T) {
 			},
 			newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
 			// Binding will be completed in the next syncClaim
-			newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation),
+			newClaimArray("claim14-2", "uid14-2", "1Gi", "", api.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
 			noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
 		},
 	}
@@ -411,8 +412,7 @@ func TestProvisionMultiSync(t *testing.T) {
 			novolumes,
 			newVolumeArray("pvc-uid12-1", "1Gi", "uid12-1", "claim12-1", api.VolumeBound, api.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned, storageutil.StorageClassAnnotation),
 			newClaimArray("claim12-1", "uid12-1", "1Gi", "", api.ClaimPending, storageutil.StorageClassAnnotation),
-			// Binding will be completed in the next syncClaim
+			newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted, annStorageProvisioner),
-			newClaimArray("claim12-1", "uid12-1", "1Gi", "pvc-uid12-1", api.ClaimBound, storageutil.StorageClassAnnotation, annBoundByController, annBindCompleted),
 			noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provision1Success}, testSyncClaim),
 		},
 	}

pkg/controller/volume/persistentvolume/pv_controller.go

@@ -117,6 +117,11 @@ const annBoundByController = "pv.kubernetes.io/bound-by-controller"
 // recognize dynamically provisioned PVs in its decisions).
 const annDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
 
+// This annotation is added to a PVC that is supposed to be dynamically
+// provisioned. Its value is name of volume plugin that is supposed to provision
+// a volume for this PVC.
+const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner"
+
 // Name of a tag attached to a real volume in cloud (e.g. AWS EBS or GCE PD)
 // with namespace of a persistent volume claim used to create this volume.
 const cloudVolumeCreatedForClaimNamespaceTag = "kubernetes.io/created-for/pvc/namespace"
@@ -1197,6 +1202,42 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
 	claimClass := storageutil.GetClaimStorageClass(claim)
 	glog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass)
 
+	plugin, storageClass, err := ctrl.findProvisionablePlugin(claim)
+	if err != nil {
+		ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error())
+		glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err)
+		// The controller will retry provisioning the volume in every
+		// syncVolume() call.
+		return
+	}
+
+	if storageClass != nil {
+		// Add provisioner annotation so external provisioners know when to start
+		newClaim, err := ctrl.setClaimProvisioner(claim, storageClass)
+		if err != nil {
+			// Save failed, the controller will retry in the next sync
+			glog.V(2).Infof("error saving claim %s: %v", claimToClaimKey(claim), err)
+			return
+		}
+		claim = newClaim
+	}
+
+	if plugin == nil {
+		// findProvisionablePlugin returned no error nor plugin.
+		// This means that an unknown provisioner is requested. Report an event
+		// and wait for the external provisioner
+		if storageClass != nil {
+			msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner)
+			ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg)
+			glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg)
+		} else {
+			glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim))
+		}
+		return
+	}
+
+	// internal provisioning
+
 	//  A previous doProvisionClaim may just have finished while we were waiting for
 	//  the locks. Check that PV (with deterministic name) hasn't been provisioned
 	//  yet.
@@ -1217,28 +1258,6 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
 		return
 	}
 
-	plugin, storageClass, err := ctrl.findProvisionablePlugin(claim)
-	if err != nil {
-		ctrl.eventRecorder.Event(claim, api.EventTypeWarning, "ProvisioningFailed", err.Error())
-		glog.V(2).Infof("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err)
-		// The controller will retry provisioning the volume in every
-		// syncVolume() call.
-		return
-	}
-	if plugin == nil {
-		// findProvisionablePlugin returned no error nor plugin.
-		// This means that an unknown provisioner is requested. Report an event
-		// and wait for the external provisioner
-		if storageClass != nil {
-			msg := fmt.Sprintf("cannot find provisioner %q, expecting that a volume for the claim is provisioned either manually or via external software", storageClass.Provisioner)
-			ctrl.eventRecorder.Event(claim, api.EventTypeNormal, "ExternalProvisioning", msg)
-			glog.V(3).Infof("provisioning claim %q: %s", claimToClaimKey(claim), msg)
-		} else {
-			glog.V(3).Infof("cannot find storage class for claim %q", claimToClaimKey(claim))
-		}
-		return
-	}
-
 	// Gather provisioning options
 	tags := make(map[string]string)
 	tags[cloudVolumeCreatedForClaimNamespaceTag] = claim.Namespace

pkg/controller/volume/persistentvolume/pv_controller_base.go

@@ -494,6 +494,36 @@ func (ctrl *PersistentVolumeController) upgradeVolumeFrom1_2(volume *api.Persist
 	return true
 }
 
+// setClaimProvisioner saves
+// claim.Annotations[annStorageProvisioner] = class.Provisioner
+func (ctrl *PersistentVolumeController) setClaimProvisioner(claim *api.PersistentVolumeClaim, class *storage.StorageClass) (*api.PersistentVolumeClaim, error) {
+	if val, ok := claim.Annotations[annDynamicallyProvisioned]; ok && val == class.Provisioner {
+		// annotation is already set, nothing to do
+		return claim, nil
+	}
+
+	// The volume from method args can be pointing to watcher cache. We must not
+	// modify these, therefore create a copy.
+	clone, err := conversion.NewCloner().DeepCopy(claim)
+	if err != nil {
+		return nil, fmt.Errorf("Error cloning pv: %v", err)
+	}
+	claimClone, ok := clone.(*api.PersistentVolumeClaim)
+	if !ok {
+		return nil, fmt.Errorf("Unexpected claim cast error : %v", claimClone)
+	}
+	api.SetMetaDataAnnotation(&claimClone.ObjectMeta, annStorageProvisioner, class.Provisioner)
+	newClaim, err := ctrl.kubeClient.Core().PersistentVolumeClaims(claim.Namespace).Update(claimClone)
+	if err != nil {
+		return newClaim, err
+	}
+	_, err = ctrl.storeClaimUpdate(newClaim)
+	if err != nil {
+		return newClaim, err
+	}
+	return newClaim, nil
+}
+
 // Stateless functions
 
 func getClaimStatusForLogging(claim *api.PersistentVolumeClaim) string {