Merge pull request #113262 from jsafrane/rework-reconstruction

Rework volume reconstruction

pkg/controller/volume/attachdetach/testing/testvolumespec.go

@@ -424,7 +424,7 @@ func (plugin *TestPlugin) NewUnmounter(name string, podUID types.UID) (volume.Un
 	return nil, nil
 }
 
-func (plugin *TestPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *TestPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	fakeVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
@@ -435,7 +435,9 @@ func (plugin *TestPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*vo
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(fakeVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(fakeVolume),
+	}, nil
 }
 
 func (plugin *TestPlugin) NewAttacher() (volume.Attacher, error) {

pkg/controller/volume/persistentvolume/framework_test.go

@@ -967,8 +967,8 @@ func (plugin *mockVolumePlugin) SupportsBulkVolumeVerification() bool {
 	return false
 }
 
-func (plugin *mockVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *mockVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
-	return nil, nil
+	return volume.ReconstructedVolume{}, nil
 }
 
 func (plugin *mockVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {

pkg/kubelet/volumemanager/cache/actual_state_of_world.go

@@ -171,6 +171,12 @@ type ActualStateOfWorld interface {
 	// SyncReconstructedVolume check the volume.outerVolumeSpecName in asw and
 	// the one populated from dsw , if they do not match, update this field from the value from dsw.
 	SyncReconstructedVolume(volumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, outerVolumeSpecName string)
+
+	// UpdateReconstructedDevicePath updates devicePath of a reconstructed volume
+	// from Node.Status.VolumesAttached. The ASW is updated only when the volume is still
+	// uncertain. If the volume got mounted in the meantime, its devicePath must have
+	// been fixed by such an update.
+	UpdateReconstructedDevicePath(volumeName v1.UniqueVolumeName, devicePath string)
 }
 
 // MountedVolume represents a volume that has successfully been mounted to a pod.
@@ -501,6 +507,24 @@ func (asw *actualStateOfWorld) MarkDeviceAsUnmounted(
 	return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceNotMounted, "", "", "")
 }
 
+func (asw *actualStateOfWorld) UpdateReconstructedDevicePath(volumeName v1.UniqueVolumeName, devicePath string) {
+	asw.Lock()
+	defer asw.Unlock()
+
+	volumeObj, volumeExists := asw.attachedVolumes[volumeName]
+	if !volumeExists {
+		return
+	}
+	if volumeObj.deviceMountState != operationexecutor.DeviceMountUncertain {
+		// Reconciler must have updated volume state, i.e. when a pod uses the volume and
+		// succeeded mounting the volume. Such update has fixed the device path.
+		return
+	}
+
+	volumeObj.devicePath = devicePath
+	asw.attachedVolumes[volumeName] = volumeObj
+}
+
 func (asw *actualStateOfWorld) GetDeviceMountState(volumeName v1.UniqueVolumeName) operationexecutor.DeviceMountState {
 	asw.RLock()
 	defer asw.RUnlock()
@@ -636,7 +660,16 @@ func (asw *actualStateOfWorld) AddPodToVolume(markVolumeOpts operationexecutor.M
 	}
 
 	podObj, podExists := volumeObj.mountedPods[podName]
-	if !podExists {
+
+	updateUncertainVolume := false
+	if podExists {
+		// Update uncertain volumes - the new markVolumeOpts may have updated information.
+		// Especially reconstructed volumes (marked as uncertain during reconstruction) need
+		// an update.
+		updateUncertainVolume = utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) && podObj.volumeMountStateForPod == operationexecutor.VolumeMountUncertain
+	}
+	if !podExists || updateUncertainVolume {
+		// Add new mountedPod or update existing one.
 		podObj = mountedPod{
 			podName:                podName,
 			podUID:                 podUID,

pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go

@@ -34,8 +34,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/component-helpers/storage/ephemeral"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/pod"
@@ -153,7 +155,10 @@ func (dswp *desiredStateOfWorldPopulator) Run(sourcesReady config.SourcesReady,
 		return done, nil
 	}, stopCh)
 	dswp.hasAddedPodsLock.Lock()
-	dswp.hasAddedPods = true
+	if !dswp.hasAddedPods {
+		klog.InfoS("Finished populating initial desired state of world")
+		dswp.hasAddedPods = true
+	}
 	dswp.hasAddedPodsLock.Unlock()
 	wait.Until(dswp.populatorLoop, dswp.loopSleepDuration, stopCh)
 }
@@ -312,8 +317,12 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes(
 		} else {
 			klog.V(4).InfoS("Added volume to desired state", "pod", klog.KObj(pod), "volumeName", podVolume.Name, "volumeSpecName", volumeSpec.Name())
 		}
-		// sync reconstructed volume
+		if !utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
-		dswp.actualStateOfWorld.SyncReconstructedVolume(uniqueVolumeName, uniquePodName, podVolume.Name)
+			// sync reconstructed volume. This is necessary only when the old-style reconstruction is still used.
+			// With reconstruct_new.go, AWS.MarkVolumeAsMounted will update the outer spec name of previously
+			// uncertain volumes.
+			dswp.actualStateOfWorld.SyncReconstructedVolume(uniqueVolumeName, uniquePodName, podVolume.Name)
+		}
 
 		dswp.checkVolumeFSResize(pod, podVolume, pvc, volumeSpec, uniquePodName, mountedVolumesForPod)
 	}

pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go

@@ -84,6 +84,9 @@ func prepareDswpWithVolume(t *testing.T) (*desiredStateOfWorldPopulator, kubepod
 }
 
 func TestFindAndAddNewPods_WithRescontructedVolume(t *testing.T) {
+	// Outer volume spec replacement is needed only when the old volume reconstruction is used
+	// (i.e. with SELinuxMountReadWriteOncePod disabled)
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, false)()
 	// create dswp
 	dswp, fakePodManager := prepareDswpWithVolume(t)
 

pkg/kubelet/volumemanager/reconciler/reconciler.go

@@ -20,141 +20,13 @@ limitations under the License.
 package reconciler
 
 import (
-	"context"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
-	"path/filepath"
-	"time"
-
-	"k8s.io/apimachinery/pkg/api/resource"
-
-	"k8s.io/klog/v2"
-	"k8s.io/mount-utils"
-	utilpath "k8s.io/utils/path"
-	utilstrings "k8s.io/utils/strings"
-
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
-	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/kubelet/config"
-	"k8s.io/kubernetes/pkg/kubelet/volumemanager/cache"
-	"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
-	volumepkg "k8s.io/kubernetes/pkg/volume"
-	"k8s.io/kubernetes/pkg/volume/util"
-	"k8s.io/kubernetes/pkg/volume/util/hostutil"
-	"k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations"
 	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
-	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
 )
 
-// Reconciler runs a periodic loop to reconcile the desired state of the world
+func (rc *reconciler) runOld(stopCh <-chan struct{}) {
-// with the actual state of the world by triggering attach, detach, mount, and
-// unmount operations.
-// Note: This is distinct from the Reconciler implemented by the attach/detach
-// controller. This reconciles state for the kubelet volume manager. That
-// reconciles state for the attach/detach controller.
-type Reconciler interface {
-	// Starts running the reconciliation loop which executes periodically, checks
-	// if volumes that should be mounted are mounted and volumes that should
-	// be unmounted are unmounted. If not, it will trigger mount/unmount
-	// operations to rectify.
-	// If attach/detach management is enabled, the manager will also check if
-	// volumes that should be attached are attached and volumes that should
-	// be detached are detached and trigger attach/detach operations as needed.
-	Run(stopCh <-chan struct{})
-
-	// StatesHasBeenSynced returns true only after syncStates process starts to sync
-	// states at least once after kubelet starts
-	StatesHasBeenSynced() bool
-}
-
-// NewReconciler returns a new instance of Reconciler.
-//
-// controllerAttachDetachEnabled - if true, indicates that the attach/detach
-//
-//	controller is responsible for managing the attach/detach operations for
-//	this node, and therefore the volume manager should not
-//
-// loopSleepDuration - the amount of time the reconciler loop sleeps between
-//
-//	successive executions
-//
-// waitForAttachTimeout - the amount of time the Mount function will wait for
-//
-//	the volume to be attached
-//
-// nodeName - the Name for this node, used by Attach and Detach methods
-// desiredStateOfWorld - cache containing the desired state of the world
-// actualStateOfWorld - cache containing the actual state of the world
-// populatorHasAddedPods - checker for whether the populator has finished
-//
-//	adding pods to the desiredStateOfWorld cache at least once after sources
-//	are all ready (before sources are ready, pods are probably missing)
-//
-// operationExecutor - used to trigger attach/detach/mount/unmount operations
-//
-//	safely (prevents more than one operation from being triggered on the same
-//	volume)
-//
-// mounter - mounter passed in from kubelet, passed down unmount path
-// hostutil - hostutil passed in from kubelet
-// volumePluginMgr - volume plugin manager passed from kubelet
-func NewReconciler(
-	kubeClient clientset.Interface,
-	controllerAttachDetachEnabled bool,
-	loopSleepDuration time.Duration,
-	waitForAttachTimeout time.Duration,
-	nodeName types.NodeName,
-	desiredStateOfWorld cache.DesiredStateOfWorld,
-	actualStateOfWorld cache.ActualStateOfWorld,
-	populatorHasAddedPods func() bool,
-	operationExecutor operationexecutor.OperationExecutor,
-	mounter mount.Interface,
-	hostutil hostutil.HostUtils,
-	volumePluginMgr *volumepkg.VolumePluginMgr,
-	kubeletPodsDir string) Reconciler {
-	return &reconciler{
-		kubeClient:                    kubeClient,
-		controllerAttachDetachEnabled: controllerAttachDetachEnabled,
-		loopSleepDuration:             loopSleepDuration,
-		waitForAttachTimeout:          waitForAttachTimeout,
-		nodeName:                      nodeName,
-		desiredStateOfWorld:           desiredStateOfWorld,
-		actualStateOfWorld:            actualStateOfWorld,
-		populatorHasAddedPods:         populatorHasAddedPods,
-		operationExecutor:             operationExecutor,
-		mounter:                       mounter,
-		hostutil:                      hostutil,
-		skippedDuringReconstruction:   map[v1.UniqueVolumeName]*globalVolumeInfo{},
-		volumePluginMgr:               volumePluginMgr,
-		kubeletPodsDir:                kubeletPodsDir,
-		timeOfLastSync:                time.Time{},
-	}
-}
-
-type reconciler struct {
-	kubeClient                    clientset.Interface
-	controllerAttachDetachEnabled bool
-	loopSleepDuration             time.Duration
-	waitForAttachTimeout          time.Duration
-	nodeName                      types.NodeName
-	desiredStateOfWorld           cache.DesiredStateOfWorld
-	actualStateOfWorld            cache.ActualStateOfWorld
-	populatorHasAddedPods         func() bool
-	operationExecutor             operationexecutor.OperationExecutor
-	mounter                       mount.Interface
-	hostutil                      hostutil.HostUtils
-	volumePluginMgr               *volumepkg.VolumePluginMgr
-	skippedDuringReconstruction   map[v1.UniqueVolumeName]*globalVolumeInfo
-	kubeletPodsDir                string
-	timeOfLastSync                time.Time
-}
-
-func (rc *reconciler) Run(stopCh <-chan struct{}) {
 	wait.Until(rc.reconciliationLoopFunc(), rc.loopSleepDuration, stopCh)
 }
 
@@ -196,84 +68,6 @@ func (rc *reconciler) reconcile() {
 	}
 }
 
-func (rc *reconciler) unmountVolumes() {
-	// Ensure volumes that should be unmounted are unmounted.
-	for _, mountedVolume := range rc.actualStateOfWorld.GetAllMountedVolumes() {
-		if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName, mountedVolume.SELinuxMountContext) {
-			// Volume is mounted, unmount it
-			klog.V(5).InfoS(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", ""))
-			err := rc.operationExecutor.UnmountVolume(
-				mountedVolume.MountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
-			if err != nil && !isExpectedError(err) {
-				klog.ErrorS(err, mountedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error())
-			}
-			if err == nil {
-				klog.InfoS(mountedVolume.GenerateMsgDetailed("operationExecutor.UnmountVolume started", ""))
-			}
-		}
-	}
-}
-
-func (rc *reconciler) mountOrAttachVolumes() {
-	// Ensure volumes that should be attached/mounted are attached/mounted.
-	for _, volumeToMount := range rc.desiredStateOfWorld.GetVolumesToMount() {
-		volMounted, devicePath, err := rc.actualStateOfWorld.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName, volumeToMount.PersistentVolumeSize, volumeToMount.SELinuxLabel)
-		volumeToMount.DevicePath = devicePath
-		if cache.IsSELinuxMountMismatchError(err) {
-			// The volume is mounted, but with an unexpected SELinux context.
-			// It will get unmounted in unmountVolumes / unmountDetachDevices and
-			// then removed from actualStateOfWorld.
-			rc.desiredStateOfWorld.AddErrorToPod(volumeToMount.PodName, err.Error())
-			continue
-		} else if cache.IsVolumeNotAttachedError(err) {
-			rc.waitForVolumeAttach(volumeToMount)
-		} else if !volMounted || cache.IsRemountRequiredError(err) {
-			rc.mountAttachedVolumes(volumeToMount, err)
-		} else if cache.IsFSResizeRequiredError(err) {
-			fsResizeRequiredErr, _ := err.(cache.FsResizeRequiredError)
-			rc.expandVolume(volumeToMount, fsResizeRequiredErr.CurrentSize)
-		}
-	}
-}
-
-func (rc *reconciler) expandVolume(volumeToMount cache.VolumeToMount, currentSize resource.Quantity) {
-	klog.V(4).InfoS(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.ExpandInUseVolume", ""), "pod", klog.KObj(volumeToMount.Pod))
-	err := rc.operationExecutor.ExpandInUseVolume(volumeToMount.VolumeToMount, rc.actualStateOfWorld, currentSize)
-
-	if err != nil && !isExpectedError(err) {
-		klog.ErrorS(err, volumeToMount.GenerateErrorDetailed("operationExecutor.ExpandInUseVolume failed", err).Error(), "pod", klog.KObj(volumeToMount.Pod))
-	}
-
-	if err == nil {
-		klog.V(4).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.ExpandInUseVolume started", ""), "pod", klog.KObj(volumeToMount.Pod))
-	}
-}
-
-func (rc *reconciler) mountAttachedVolumes(volumeToMount cache.VolumeToMount, podExistError error) {
-	// Volume is not mounted, or is already mounted, but requires remounting
-	remountingLogStr := ""
-	isRemount := cache.IsRemountRequiredError(podExistError)
-	if isRemount {
-		remountingLogStr = "Volume is already mounted to pod, but remount was requested."
-	}
-	klog.V(4).InfoS(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr), "pod", klog.KObj(volumeToMount.Pod))
-	err := rc.operationExecutor.MountVolume(
-		rc.waitForAttachTimeout,
-		volumeToMount.VolumeToMount,
-		rc.actualStateOfWorld,
-		isRemount)
-	if err != nil && !isExpectedError(err) {
-		klog.ErrorS(err, volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.MountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error(), "pod", klog.KObj(volumeToMount.Pod))
-	}
-	if err == nil {
-		if remountingLogStr == "" {
-			klog.V(1).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr), "pod", klog.KObj(volumeToMount.Pod))
-		} else {
-			klog.V(5).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr), "pod", klog.KObj(volumeToMount.Pod))
-		}
-	}
-}
-
 // processReconstructedVolumes checks volumes which were skipped during the reconstruction
 // process because it was assumed that since these volumes were present in DSOW they would get
 // mounted correctly and make it into ASOW.
@@ -336,528 +130,3 @@ func (rc *reconciler) processReconstructedVolumes() {
 		}
 	}
 }
-
-func (rc *reconciler) waitForVolumeAttach(volumeToMount cache.VolumeToMount) {
-	if rc.controllerAttachDetachEnabled || !volumeToMount.PluginIsAttachable {
-		//// lets not spin a goroutine and unnecessarily trigger exponential backoff if this happens
-		if volumeToMount.PluginIsAttachable && !volumeToMount.ReportedInUse {
-			klog.V(5).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.VerifyControllerAttachedVolume failed", " volume not marked in-use"), "pod", klog.KObj(volumeToMount.Pod))
-			return
-		}
-		// Volume is not attached (or doesn't implement attacher), kubelet attach is disabled, wait
-		// for controller to finish attaching volume.
-		klog.V(5).InfoS(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.VerifyControllerAttachedVolume", ""), "pod", klog.KObj(volumeToMount.Pod))
-		err := rc.operationExecutor.VerifyControllerAttachedVolume(
-			volumeToMount.VolumeToMount,
-			rc.nodeName,
-			rc.actualStateOfWorld)
-		if err != nil && !isExpectedError(err) {
-			klog.ErrorS(err, volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.VerifyControllerAttachedVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error(), "pod", klog.KObj(volumeToMount.Pod))
-		}
-		if err == nil {
-			klog.InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.VerifyControllerAttachedVolume started", ""), "pod", klog.KObj(volumeToMount.Pod))
-		}
-	} else {
-		// Volume is not attached to node, kubelet attach is enabled, volume implements an attacher,
-		// so attach it
-		volumeToAttach := operationexecutor.VolumeToAttach{
-			VolumeName: volumeToMount.VolumeName,
-			VolumeSpec: volumeToMount.VolumeSpec,
-			NodeName:   rc.nodeName,
-		}
-		klog.V(5).InfoS(volumeToAttach.GenerateMsgDetailed("Starting operationExecutor.AttachVolume", ""), "pod", klog.KObj(volumeToMount.Pod))
-		err := rc.operationExecutor.AttachVolume(volumeToAttach, rc.actualStateOfWorld)
-		if err != nil && !isExpectedError(err) {
-			klog.ErrorS(err, volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.AttachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error(), "pod", klog.KObj(volumeToMount.Pod))
-		}
-		if err == nil {
-			klog.InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.AttachVolume started", ""), "pod", klog.KObj(volumeToMount.Pod))
-		}
-	}
-}
-
-func (rc *reconciler) unmountDetachDevices() {
-	for _, attachedVolume := range rc.actualStateOfWorld.GetUnmountedVolumes() {
-		// Check IsOperationPending to avoid marking a volume as detached if it's in the process of mounting.
-		if !rc.desiredStateOfWorld.VolumeExists(attachedVolume.VolumeName, attachedVolume.SELinuxMountContext) &&
-			!rc.operationExecutor.IsOperationPending(attachedVolume.VolumeName, nestedpendingoperations.EmptyUniquePodName, nestedpendingoperations.EmptyNodeName) {
-			if attachedVolume.DeviceMayBeMounted() {
-				// Volume is globally mounted to device, unmount it
-				klog.V(5).InfoS(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", ""))
-				err := rc.operationExecutor.UnmountDevice(
-					attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.hostutil)
-				if err != nil && !isExpectedError(err) {
-					klog.ErrorS(err, attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountDevice failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error())
-				}
-				if err == nil {
-					klog.InfoS(attachedVolume.GenerateMsgDetailed("operationExecutor.UnmountDevice started", ""))
-				}
-			} else {
-				// Volume is attached to node, detach it
-				// Kubelet not responsible for detaching or this volume has a non-attachable volume plugin.
-				if rc.controllerAttachDetachEnabled || !attachedVolume.PluginIsAttachable {
-					rc.actualStateOfWorld.MarkVolumeAsDetached(attachedVolume.VolumeName, attachedVolume.NodeName)
-					klog.InfoS(attachedVolume.GenerateMsgDetailed("Volume detached", fmt.Sprintf("DevicePath %q", attachedVolume.DevicePath)))
-				} else {
-					// Only detach if kubelet detach is enabled
-					klog.V(5).InfoS(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.DetachVolume", ""))
-					err := rc.operationExecutor.DetachVolume(
-						attachedVolume.AttachedVolume, false /* verifySafeToDetach */, rc.actualStateOfWorld)
-					if err != nil && !isExpectedError(err) {
-						klog.ErrorS(err, attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.DetachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error())
-					}
-					if err == nil {
-						klog.InfoS(attachedVolume.GenerateMsgDetailed("operationExecutor.DetachVolume started", ""))
-					}
-				}
-			}
-		}
-	}
-}
-
-// sync process tries to observe the real world by scanning all pods' volume directories from the disk.
-// If the actual and desired state of worlds are not consistent with the observed world, it means that some
-// mounted volumes are left out probably during kubelet restart. This process will reconstruct
-// the volumes and update the actual and desired states. For the volumes that cannot support reconstruction,
-// it will try to clean up the mount paths with operation executor.
-func (rc *reconciler) sync() {
-	defer rc.updateLastSyncTime()
-	rc.syncStates(rc.kubeletPodsDir)
-}
-
-func (rc *reconciler) updateLastSyncTime() {
-	rc.timeOfLastSync = time.Now()
-}
-
-func (rc *reconciler) StatesHasBeenSynced() bool {
-	return !rc.timeOfLastSync.IsZero()
-}
-
-type podVolume struct {
-	podName        volumetypes.UniquePodName
-	volumeSpecName string
-	volumePath     string
-	pluginName     string
-	volumeMode     v1.PersistentVolumeMode
-}
-
-type reconstructedVolume struct {
-	volumeName          v1.UniqueVolumeName
-	podName             volumetypes.UniquePodName
-	volumeSpec          *volumepkg.Spec
-	outerVolumeSpecName string
-	pod                 *v1.Pod
-	volumeGidValue      string
-	devicePath          string
-	mounter             volumepkg.Mounter
-	deviceMounter       volumepkg.DeviceMounter
-	blockVolumeMapper   volumepkg.BlockVolumeMapper
-}
-
-// globalVolumeInfo stores reconstructed volume information
-// for each pod that was using that volume.
-type globalVolumeInfo struct {
-	volumeName        v1.UniqueVolumeName
-	volumeSpec        *volumepkg.Spec
-	devicePath        string
-	mounter           volumepkg.Mounter
-	deviceMounter     volumepkg.DeviceMounter
-	blockVolumeMapper volumepkg.BlockVolumeMapper
-	podVolumes        map[volumetypes.UniquePodName]*reconstructedVolume
-}
-
-func (gvi *globalVolumeInfo) addPodVolume(rcv *reconstructedVolume) {
-	if gvi.podVolumes == nil {
-		gvi.podVolumes = map[volumetypes.UniquePodName]*reconstructedVolume{}
-	}
-	gvi.podVolumes[rcv.podName] = rcv
-}
-
-// syncStates scans the volume directories under the given pod directory.
-// If the volume is not in desired state of world, this function will reconstruct
-// the volume related information and put it in both the actual and desired state of worlds.
-// For some volume plugins that cannot support reconstruction, it will clean up the existing
-// mount points since the volume is no long needed (removed from desired state)
-func (rc *reconciler) syncStates(kubeletPodDir string) {
-	// Get volumes information by reading the pod's directory
-	podVolumes, err := getVolumesFromPodDir(kubeletPodDir)
-	if err != nil {
-		klog.ErrorS(err, "Cannot get volumes from disk, skip sync states for volume reconstruction")
-		return
-	}
-	volumesNeedUpdate := make(map[v1.UniqueVolumeName]*globalVolumeInfo)
-	volumeNeedReport := []v1.UniqueVolumeName{}
-	for _, volume := range podVolumes {
-		if rc.actualStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
-			klog.V(4).InfoS("Volume exists in actual state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
-			// There is nothing to reconstruct
-			continue
-		}
-		volumeInDSW := rc.desiredStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName)
-
-		reconstructedVolume, err := rc.reconstructVolume(volume)
-		if err != nil {
-			if volumeInDSW {
-				// Some pod needs the volume, don't clean it up and hope that
-				// reconcile() calls SetUp and reconstructs the volume in ASW.
-				klog.V(4).InfoS("Volume exists in desired state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
-				continue
-			}
-			// No pod needs the volume.
-			klog.InfoS("Could not construct volume information, cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName, "err", err)
-			rc.cleanupMounts(volume)
-			continue
-		}
-		gvl := &globalVolumeInfo{
-			volumeName:        reconstructedVolume.volumeName,
-			volumeSpec:        reconstructedVolume.volumeSpec,
-			devicePath:        reconstructedVolume.devicePath,
-			deviceMounter:     reconstructedVolume.deviceMounter,
-			blockVolumeMapper: reconstructedVolume.blockVolumeMapper,
-			mounter:           reconstructedVolume.mounter,
-		}
-		if cachedInfo, ok := volumesNeedUpdate[reconstructedVolume.volumeName]; ok {
-			gvl = cachedInfo
-		}
-		gvl.addPodVolume(reconstructedVolume)
-		if volumeInDSW {
-			// Some pod needs the volume. And it exists on disk. Some previous
-			// kubelet must have created the directory, therefore it must have
-			// reported the volume as in use. Mark the volume as in use also in
-			// this new kubelet so reconcile() calls SetUp and re-mounts the
-			// volume if it's necessary.
-			volumeNeedReport = append(volumeNeedReport, reconstructedVolume.volumeName)
-			rc.skippedDuringReconstruction[reconstructedVolume.volumeName] = gvl
-			klog.V(4).InfoS("Volume exists in desired state, marking as InUse", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
-			continue
-		}
-		// There is no pod that uses the volume.
-		if rc.operationExecutor.IsOperationPending(reconstructedVolume.volumeName, nestedpendingoperations.EmptyUniquePodName, nestedpendingoperations.EmptyNodeName) {
-			klog.InfoS("Volume is in pending operation, skip cleaning up mounts")
-		}
-		klog.V(2).InfoS("Reconciler sync states: could not find pod information in desired state, update it in actual state", "reconstructedVolume", reconstructedVolume)
-		volumesNeedUpdate[reconstructedVolume.volumeName] = gvl
-	}
-
-	if len(volumesNeedUpdate) > 0 {
-		if err = rc.updateStates(volumesNeedUpdate); err != nil {
-			klog.ErrorS(err, "Error occurred during reconstruct volume from disk")
-		}
-	}
-	if len(volumeNeedReport) > 0 {
-		rc.desiredStateOfWorld.MarkVolumesReportedInUse(volumeNeedReport)
-	}
-}
-
-func (rc *reconciler) cleanupMounts(volume podVolume) {
-	klog.V(2).InfoS("Reconciler sync states: could not find volume information in desired state, clean up the mount points", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
-	mountedVolume := operationexecutor.MountedVolume{
-		PodName:             volume.podName,
-		VolumeName:          v1.UniqueVolumeName(volume.volumeSpecName),
-		InnerVolumeSpecName: volume.volumeSpecName,
-		PluginName:          volume.pluginName,
-		PodUID:              types.UID(volume.podName),
-	}
-	// TODO: Currently cleanupMounts only includes UnmountVolume operation. In the next PR, we will add
-	// to unmount both volume and device in the same routine.
-	err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
-	if err != nil {
-		klog.ErrorS(err, mountedVolume.GenerateErrorDetailed("volumeHandler.UnmountVolumeHandler for UnmountVolume failed", err).Error())
-		return
-	}
-}
-
-// Reconstruct volume data structure by reading the pod's volume directories
-func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, error) {
-	// plugin initializations
-	plugin, err := rc.volumePluginMgr.FindPluginByName(volume.pluginName)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create pod object
-	pod := &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			UID: types.UID(volume.podName),
-		},
-	}
-	mapperPlugin, err := rc.volumePluginMgr.FindMapperPluginByName(volume.pluginName)
-	if err != nil {
-		return nil, err
-	}
-	if volume.volumeMode == v1.PersistentVolumeBlock && mapperPlugin == nil {
-		return nil, fmt.Errorf("could not find block volume plugin %q (spec.Name: %q) pod %q (UID: %q)", volume.pluginName, volume.volumeSpecName, volume.podName, pod.UID)
-	}
-
-	volumeSpec, err := rc.operationExecutor.ReconstructVolumeOperation(
-		volume.volumeMode,
-		plugin,
-		mapperPlugin,
-		pod.UID,
-		volume.podName,
-		volume.volumeSpecName,
-		volume.volumePath,
-		volume.pluginName)
-	if err != nil {
-		return nil, err
-	}
-
-	// We have to find the plugins by volume spec (NOT by plugin name) here
-	// in order to correctly reconstruct ephemeral volume types.
-	// Searching by spec checks whether the volume is actually attachable
-	// (i.e. has a PV) whereas searching by plugin name can only tell whether
-	// the plugin supports attachable volumes.
-	attachablePlugin, err := rc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec)
-	if err != nil {
-		return nil, err
-	}
-	deviceMountablePlugin, err := rc.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	var uniqueVolumeName v1.UniqueVolumeName
-	if attachablePlugin != nil || deviceMountablePlugin != nil {
-		uniqueVolumeName, err = util.GetUniqueVolumeNameFromSpec(plugin, volumeSpec)
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		uniqueVolumeName = util.GetUniqueVolumeNameFromSpecWithPod(volume.podName, plugin, volumeSpec)
-	}
-
-	var volumeMapper volumepkg.BlockVolumeMapper
-	var volumeMounter volumepkg.Mounter
-	var deviceMounter volumepkg.DeviceMounter
-	// Path to the mount or block device to check
-	var checkPath string
-
-	if volume.volumeMode == v1.PersistentVolumeBlock {
-		var newMapperErr error
-		volumeMapper, newMapperErr = mapperPlugin.NewBlockVolumeMapper(
-			volumeSpec,
-			pod,
-			volumepkg.VolumeOptions{})
-		if newMapperErr != nil {
-			return nil, fmt.Errorf(
-				"reconstructVolume.NewBlockVolumeMapper failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
-				uniqueVolumeName,
-				volumeSpec.Name(),
-				volume.podName,
-				pod.UID,
-				newMapperErr)
-		}
-		mapDir, linkName := volumeMapper.GetPodDeviceMapPath()
-		checkPath = filepath.Join(mapDir, linkName)
-	} else {
-		var err error
-		volumeMounter, err = plugin.NewMounter(
-			volumeSpec,
-			pod,
-			volumepkg.VolumeOptions{})
-		if err != nil {
-			return nil, fmt.Errorf(
-				"reconstructVolume.NewMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
-				uniqueVolumeName,
-				volumeSpec.Name(),
-				volume.podName,
-				pod.UID,
-				err)
-		}
-		checkPath = volumeMounter.GetPath()
-		if deviceMountablePlugin != nil {
-			deviceMounter, err = deviceMountablePlugin.NewDeviceMounter()
-			if err != nil {
-				return nil, fmt.Errorf("reconstructVolume.NewDeviceMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
-					uniqueVolumeName,
-					volumeSpec.Name(),
-					volume.podName,
-					pod.UID,
-					err)
-			}
-		}
-	}
-
-	// Check existence of mount point for filesystem volume or symbolic link for block volume
-	isExist, checkErr := rc.operationExecutor.CheckVolumeExistenceOperation(volumeSpec, checkPath, volumeSpec.Name(), rc.mounter, uniqueVolumeName, volume.podName, pod.UID, attachablePlugin)
-	if checkErr != nil {
-		return nil, checkErr
-	}
-	// If mount or symlink doesn't exist, volume reconstruction should be failed
-	if !isExist {
-		return nil, fmt.Errorf("volume: %q is not mounted", uniqueVolumeName)
-	}
-
-	reconstructedVolume := &reconstructedVolume{
-		volumeName: uniqueVolumeName,
-		podName:    volume.podName,
-		volumeSpec: volumeSpec,
-		// volume.volumeSpecName is actually InnerVolumeSpecName. It will not be used
-		// for volume cleanup.
-		// in case pod is added back to desired state, outerVolumeSpecName will be updated from dsw information.
-		// See issue #103143 and its fix for details.
-		outerVolumeSpecName: volume.volumeSpecName,
-		pod:                 pod,
-		deviceMounter:       deviceMounter,
-		volumeGidValue:      "",
-		// devicePath is updated during updateStates() by checking node status's VolumesAttached data.
-		// TODO: get device path directly from the volume mount path.
-		devicePath:        "",
-		mounter:           volumeMounter,
-		blockVolumeMapper: volumeMapper,
-	}
-	return reconstructedVolume, nil
-}
-
-// updateDevicePath gets the node status to retrieve volume device path information.
-func (rc *reconciler) updateDevicePath(volumesNeedUpdate map[v1.UniqueVolumeName]*globalVolumeInfo) {
-	node, fetchErr := rc.kubeClient.CoreV1().Nodes().Get(context.TODO(), string(rc.nodeName), metav1.GetOptions{})
-	if fetchErr != nil {
-		klog.ErrorS(fetchErr, "UpdateStates in reconciler: could not get node status with error")
-	} else {
-		for _, attachedVolume := range node.Status.VolumesAttached {
-			if volume, exists := volumesNeedUpdate[attachedVolume.Name]; exists {
-				volume.devicePath = attachedVolume.DevicePath
-				volumesNeedUpdate[attachedVolume.Name] = volume
-				klog.V(4).InfoS("Update devicePath from node status for volume", "volumeName", attachedVolume.Name, "path", volume.devicePath)
-			}
-		}
-	}
-}
-
-// getDeviceMountPath returns device mount path for block volume which
-// implements BlockVolumeMapper or filesystem volume which implements
-// DeviceMounter
-func getDeviceMountPath(gvi *globalVolumeInfo) (string, error) {
-	if gvi.blockVolumeMapper != nil {
-		// for block gvi, we return its global map path
-		return gvi.blockVolumeMapper.GetGlobalMapPath(gvi.volumeSpec)
-	} else if gvi.deviceMounter != nil {
-		// for filesystem gvi, we return its device mount path if the plugin implements DeviceMounter
-		return gvi.deviceMounter.GetDeviceMountPath(gvi.volumeSpec)
-	} else {
-		return "", fmt.Errorf("blockVolumeMapper or deviceMounter required")
-	}
-}
-
-func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*globalVolumeInfo) error {
-	// Get the node status to retrieve volume device path information.
-	// Skip reporting devicePath in node objects if kubeClient is nil.
-	// In standalone mode, kubelet is not expected to mount any attachable volume types or secret, configmaps etc.
-	if rc.kubeClient != nil {
-		rc.updateDevicePath(volumesNeedUpdate)
-	}
-
-	for _, gvl := range volumesNeedUpdate {
-		err := rc.actualStateOfWorld.MarkVolumeAsAttached(
-			//TODO: the devicePath might not be correct for some volume plugins: see issue #54108
-			gvl.volumeName, gvl.volumeSpec, "" /* nodeName */, gvl.devicePath)
-		if err != nil {
-			klog.ErrorS(err, "Could not add volume information to actual state of world", "volumeName", gvl.volumeName)
-			continue
-		}
-		for _, volume := range gvl.podVolumes {
-			err = rc.markVolumeState(volume, operationexecutor.VolumeMounted)
-			if err != nil {
-				klog.ErrorS(err, "Could not add pod to volume information to actual state of world", "pod", klog.KObj(volume.pod))
-				continue
-			}
-			klog.V(4).InfoS("Volume is marked as mounted and added into the actual state", "pod", klog.KObj(volume.pod), "podName", volume.podName, "volumeName", volume.volumeName)
-		}
-		// If the volume has device to mount, we mark its device as mounted.
-		if gvl.deviceMounter != nil || gvl.blockVolumeMapper != nil {
-			deviceMountPath, err := getDeviceMountPath(gvl)
-			if err != nil {
-				klog.ErrorS(err, "Could not find device mount path for volume", "volumeName", gvl.volumeName)
-				continue
-			}
-			// TODO(jsafrane): add reconstructed SELinux context
-			err = rc.actualStateOfWorld.MarkDeviceAsMounted(gvl.volumeName, gvl.devicePath, deviceMountPath, "")
-			if err != nil {
-				klog.ErrorS(err, "Could not mark device is mounted to actual state of world", "volume", gvl.volumeName)
-				continue
-			}
-			klog.V(4).InfoS("Volume is marked device as mounted and added into the actual state", "volumeName", gvl.volumeName)
-		}
-	}
-	return nil
-}
-
-func (rc *reconciler) markVolumeState(volume *reconstructedVolume, volumeState operationexecutor.VolumeMountState) error {
-	markVolumeOpts := operationexecutor.MarkVolumeOpts{
-		PodName:             volume.podName,
-		PodUID:              types.UID(volume.podName),
-		VolumeName:          volume.volumeName,
-		Mounter:             volume.mounter,
-		BlockVolumeMapper:   volume.blockVolumeMapper,
-		OuterVolumeSpecName: volume.outerVolumeSpecName,
-		VolumeGidVolume:     volume.volumeGidValue,
-		VolumeSpec:          volume.volumeSpec,
-		VolumeMountState:    volumeState,
-	}
-	err := rc.actualStateOfWorld.MarkVolumeAsMounted(markVolumeOpts)
-	return err
-}
-
-// getVolumesFromPodDir scans through the volumes directories under the given pod directory.
-// It returns a list of pod volume information including pod's uid, volume's plugin name, mount path,
-// and volume spec name.
-func getVolumesFromPodDir(podDir string) ([]podVolume, error) {
-	podsDirInfo, err := os.ReadDir(podDir)
-	if err != nil {
-		return nil, err
-	}
-	volumes := []podVolume{}
-	for i := range podsDirInfo {
-		if !podsDirInfo[i].IsDir() {
-			continue
-		}
-		podName := podsDirInfo[i].Name()
-		podDir := path.Join(podDir, podName)
-
-		// Find filesystem volume information
-		// ex. filesystem volume: /pods/{podUid}/volume/{escapeQualifiedPluginName}/{volumeName}
-		volumesDirs := map[v1.PersistentVolumeMode]string{
-			v1.PersistentVolumeFilesystem: path.Join(podDir, config.DefaultKubeletVolumesDirName),
-		}
-		// Find block volume information
-		// ex. block volume: /pods/{podUid}/volumeDevices/{escapeQualifiedPluginName}/{volumeName}
-		volumesDirs[v1.PersistentVolumeBlock] = path.Join(podDir, config.DefaultKubeletVolumeDevicesDirName)
-
-		for volumeMode, volumesDir := range volumesDirs {
-			var volumesDirInfo []fs.DirEntry
-			if volumesDirInfo, err = os.ReadDir(volumesDir); err != nil {
-				// Just skip the loop because given volumesDir doesn't exist depending on volumeMode
-				continue
-			}
-			for _, volumeDir := range volumesDirInfo {
-				pluginName := volumeDir.Name()
-				volumePluginPath := path.Join(volumesDir, pluginName)
-				volumePluginDirs, err := utilpath.ReadDirNoStat(volumePluginPath)
-				if err != nil {
-					klog.ErrorS(err, "Could not read volume plugin directory", "volumePluginPath", volumePluginPath)
-					continue
-				}
-				unescapePluginName := utilstrings.UnescapeQualifiedName(pluginName)
-				for _, volumeName := range volumePluginDirs {
-					volumePath := path.Join(volumePluginPath, volumeName)
-					klog.V(5).InfoS("Volume path from volume plugin directory", "podName", podName, "volumePath", volumePath)
-					volumes = append(volumes, podVolume{
-						podName:        volumetypes.UniquePodName(podName),
-						volumeSpecName: volumeName,
-						volumePath:     volumePath,
-						pluginName:     unescapePluginName,
-						volumeMode:     volumeMode,
-					})
-				}
-			}
-		}
-	}
-	klog.V(4).InfoS("Get volumes from pod directory", "path", podDir, "volumes", volumes)
-	return volumes, nil
-}
-
-// ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected.
-func isExpectedError(err error) bool {
-	return nestedpendingoperations.IsAlreadyExists(err) || exponentialbackoff.IsExponentialBackoff(err) || operationexecutor.IsMountFailedPreconditionError(err)
-}

pkg/kubelet/volumemanager/reconciler/reconciler_common.go

@@ -0,0 +1,316 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reconciler
+
+import (
+	"fmt"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/kubelet/volumemanager/cache"
+	"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
+	volumepkg "k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util/hostutil"
+	"k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations"
+	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
+	"k8s.io/mount-utils"
+)
+
+// Reconciler runs a periodic loop to reconcile the desired state of the world
+// with the actual state of the world by triggering attach, detach, mount, and
+// unmount operations.
+// Note: This is distinct from the Reconciler implemented by the attach/detach
+// controller. This reconciles state for the kubelet volume manager. That
+// reconciles state for the attach/detach controller.
+type Reconciler interface {
+	// Starts running the reconciliation loop which executes periodically, checks
+	// if volumes that should be mounted are mounted and volumes that should
+	// be unmounted are unmounted. If not, it will trigger mount/unmount
+	// operations to rectify.
+	// If attach/detach management is enabled, the manager will also check if
+	// volumes that should be attached are attached and volumes that should
+	// be detached are detached and trigger attach/detach operations as needed.
+	Run(stopCh <-chan struct{})
+
+	// StatesHasBeenSynced returns true only after syncStates process starts to sync
+	// states at least once after kubelet starts
+	StatesHasBeenSynced() bool
+}
+
+// NewReconciler returns a new instance of Reconciler.
+//
+// controllerAttachDetachEnabled - if true, indicates that the attach/detach
+//
+//	controller is responsible for managing the attach/detach operations for
+//	this node, and therefore the volume manager should not
+//
+// loopSleepDuration - the amount of time the reconciler loop sleeps between
+//
+//	successive executions
+//
+// waitForAttachTimeout - the amount of time the Mount function will wait for
+//
+//	the volume to be attached
+//
+// nodeName - the Name for this node, used by Attach and Detach methods
+// desiredStateOfWorld - cache containing the desired state of the world
+// actualStateOfWorld - cache containing the actual state of the world
+// populatorHasAddedPods - checker for whether the populator has finished
+//
+//	adding pods to the desiredStateOfWorld cache at least once after sources
+//	are all ready (before sources are ready, pods are probably missing)
+//
+// operationExecutor - used to trigger attach/detach/mount/unmount operations
+//
+//	safely (prevents more than one operation from being triggered on the same
+//	volume)
+//
+// mounter - mounter passed in from kubelet, passed down unmount path
+// hostutil - hostutil passed in from kubelet
+// volumePluginMgr - volume plugin manager passed from kubelet
+func NewReconciler(
+	kubeClient clientset.Interface,
+	controllerAttachDetachEnabled bool,
+	loopSleepDuration time.Duration,
+	waitForAttachTimeout time.Duration,
+	nodeName types.NodeName,
+	desiredStateOfWorld cache.DesiredStateOfWorld,
+	actualStateOfWorld cache.ActualStateOfWorld,
+	populatorHasAddedPods func() bool,
+	operationExecutor operationexecutor.OperationExecutor,
+	mounter mount.Interface,
+	hostutil hostutil.HostUtils,
+	volumePluginMgr *volumepkg.VolumePluginMgr,
+	kubeletPodsDir string) Reconciler {
+	return &reconciler{
+		kubeClient:                    kubeClient,
+		controllerAttachDetachEnabled: controllerAttachDetachEnabled,
+		loopSleepDuration:             loopSleepDuration,
+		waitForAttachTimeout:          waitForAttachTimeout,
+		nodeName:                      nodeName,
+		desiredStateOfWorld:           desiredStateOfWorld,
+		actualStateOfWorld:            actualStateOfWorld,
+		populatorHasAddedPods:         populatorHasAddedPods,
+		operationExecutor:             operationExecutor,
+		mounter:                       mounter,
+		hostutil:                      hostutil,
+		skippedDuringReconstruction:   map[v1.UniqueVolumeName]*globalVolumeInfo{},
+		volumePluginMgr:               volumePluginMgr,
+		kubeletPodsDir:                kubeletPodsDir,
+		timeOfLastSync:                time.Time{},
+		volumesFailedReconstruction:   make([]podVolume, 0),
+		volumesNeedDevicePath:         make([]v1.UniqueVolumeName, 0),
+		volumesNeedReportedInUse:      make([]v1.UniqueVolumeName, 0),
+	}
+}
+
+type reconciler struct {
+	kubeClient                    clientset.Interface
+	controllerAttachDetachEnabled bool
+	loopSleepDuration             time.Duration
+	waitForAttachTimeout          time.Duration
+	nodeName                      types.NodeName
+	desiredStateOfWorld           cache.DesiredStateOfWorld
+	actualStateOfWorld            cache.ActualStateOfWorld
+	populatorHasAddedPods         func() bool
+	operationExecutor             operationexecutor.OperationExecutor
+	mounter                       mount.Interface
+	hostutil                      hostutil.HostUtils
+	volumePluginMgr               *volumepkg.VolumePluginMgr
+	skippedDuringReconstruction   map[v1.UniqueVolumeName]*globalVolumeInfo
+	kubeletPodsDir                string
+	timeOfLastSync                time.Time
+	volumesFailedReconstruction   []podVolume
+	volumesNeedDevicePath         []v1.UniqueVolumeName
+	volumesNeedReportedInUse      []v1.UniqueVolumeName
+}
+
+func (rc *reconciler) Run(stopCh <-chan struct{}) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
+		rc.runNew(stopCh)
+		return
+	}
+
+	rc.runOld(stopCh)
+}
+
+func (rc *reconciler) unmountVolumes() {
+	// Ensure volumes that should be unmounted are unmounted.
+	for _, mountedVolume := range rc.actualStateOfWorld.GetAllMountedVolumes() {
+		if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName, mountedVolume.SELinuxMountContext) {
+			// Volume is mounted, unmount it
+			klog.V(5).InfoS(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", ""))
+			err := rc.operationExecutor.UnmountVolume(
+				mountedVolume.MountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
+			if err != nil && !isExpectedError(err) {
+				klog.ErrorS(err, mountedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error())
+			}
+			if err == nil {
+				klog.InfoS(mountedVolume.GenerateMsgDetailed("operationExecutor.UnmountVolume started", ""))
+			}
+		}
+	}
+}
+
+func (rc *reconciler) mountOrAttachVolumes() {
+	// Ensure volumes that should be attached/mounted are attached/mounted.
+	for _, volumeToMount := range rc.desiredStateOfWorld.GetVolumesToMount() {
+		volMounted, devicePath, err := rc.actualStateOfWorld.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName, volumeToMount.PersistentVolumeSize, volumeToMount.SELinuxLabel)
+		volumeToMount.DevicePath = devicePath
+		if cache.IsSELinuxMountMismatchError(err) {
+			// The volume is mounted, but with an unexpected SELinux context.
+			// It will get unmounted in unmountVolumes / unmountDetachDevices and
+			// then removed from actualStateOfWorld.
+			rc.desiredStateOfWorld.AddErrorToPod(volumeToMount.PodName, err.Error())
+			continue
+		} else if cache.IsVolumeNotAttachedError(err) {
+			rc.waitForVolumeAttach(volumeToMount)
+		} else if !volMounted || cache.IsRemountRequiredError(err) {
+			rc.mountAttachedVolumes(volumeToMount, err)
+		} else if cache.IsFSResizeRequiredError(err) {
+			fsResizeRequiredErr, _ := err.(cache.FsResizeRequiredError)
+			rc.expandVolume(volumeToMount, fsResizeRequiredErr.CurrentSize)
+		}
+	}
+}
+
+func (rc *reconciler) expandVolume(volumeToMount cache.VolumeToMount, currentSize resource.Quantity) {
+	klog.V(4).InfoS(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.ExpandInUseVolume", ""), "pod", klog.KObj(volumeToMount.Pod))
+	err := rc.operationExecutor.ExpandInUseVolume(volumeToMount.VolumeToMount, rc.actualStateOfWorld, currentSize)
+
+	if err != nil && !isExpectedError(err) {
+		klog.ErrorS(err, volumeToMount.GenerateErrorDetailed("operationExecutor.ExpandInUseVolume failed", err).Error(), "pod", klog.KObj(volumeToMount.Pod))
+	}
+
+	if err == nil {
+		klog.V(4).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.ExpandInUseVolume started", ""), "pod", klog.KObj(volumeToMount.Pod))
+	}
+}
+
+func (rc *reconciler) mountAttachedVolumes(volumeToMount cache.VolumeToMount, podExistError error) {
+	// Volume is not mounted, or is already mounted, but requires remounting
+	remountingLogStr := ""
+	isRemount := cache.IsRemountRequiredError(podExistError)
+	if isRemount {
+		remountingLogStr = "Volume is already mounted to pod, but remount was requested."
+	}
+	klog.V(4).InfoS(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr), "pod", klog.KObj(volumeToMount.Pod))
+	err := rc.operationExecutor.MountVolume(
+		rc.waitForAttachTimeout,
+		volumeToMount.VolumeToMount,
+		rc.actualStateOfWorld,
+		isRemount)
+	if err != nil && !isExpectedError(err) {
+		klog.ErrorS(err, volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.MountVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error(), "pod", klog.KObj(volumeToMount.Pod))
+	}
+	if err == nil {
+		if remountingLogStr == "" {
+			klog.V(1).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr), "pod", klog.KObj(volumeToMount.Pod))
+		} else {
+			klog.V(5).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.MountVolume started", remountingLogStr), "pod", klog.KObj(volumeToMount.Pod))
+		}
+	}
+}
+
+func (rc *reconciler) waitForVolumeAttach(volumeToMount cache.VolumeToMount) {
+	if rc.controllerAttachDetachEnabled || !volumeToMount.PluginIsAttachable {
+		//// lets not spin a goroutine and unnecessarily trigger exponential backoff if this happens
+		if volumeToMount.PluginIsAttachable && !volumeToMount.ReportedInUse {
+			klog.V(5).InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.VerifyControllerAttachedVolume failed", " volume not marked in-use"), "pod", klog.KObj(volumeToMount.Pod))
+			return
+		}
+		// Volume is not attached (or doesn't implement attacher), kubelet attach is disabled, wait
+		// for controller to finish attaching volume.
+		klog.V(5).InfoS(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.VerifyControllerAttachedVolume", ""), "pod", klog.KObj(volumeToMount.Pod))
+		err := rc.operationExecutor.VerifyControllerAttachedVolume(
+			volumeToMount.VolumeToMount,
+			rc.nodeName,
+			rc.actualStateOfWorld)
+		if err != nil && !isExpectedError(err) {
+			klog.ErrorS(err, volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.VerifyControllerAttachedVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error(), "pod", klog.KObj(volumeToMount.Pod))
+		}
+		if err == nil {
+			klog.InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.VerifyControllerAttachedVolume started", ""), "pod", klog.KObj(volumeToMount.Pod))
+		}
+	} else {
+		// Volume is not attached to node, kubelet attach is enabled, volume implements an attacher,
+		// so attach it
+		volumeToAttach := operationexecutor.VolumeToAttach{
+			VolumeName: volumeToMount.VolumeName,
+			VolumeSpec: volumeToMount.VolumeSpec,
+			NodeName:   rc.nodeName,
+		}
+		klog.V(5).InfoS(volumeToAttach.GenerateMsgDetailed("Starting operationExecutor.AttachVolume", ""), "pod", klog.KObj(volumeToMount.Pod))
+		err := rc.operationExecutor.AttachVolume(volumeToAttach, rc.actualStateOfWorld)
+		if err != nil && !isExpectedError(err) {
+			klog.ErrorS(err, volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.AttachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error(), "pod", klog.KObj(volumeToMount.Pod))
+		}
+		if err == nil {
+			klog.InfoS(volumeToMount.GenerateMsgDetailed("operationExecutor.AttachVolume started", ""), "pod", klog.KObj(volumeToMount.Pod))
+		}
+	}
+}
+
+func (rc *reconciler) unmountDetachDevices() {
+	for _, attachedVolume := range rc.actualStateOfWorld.GetUnmountedVolumes() {
+		// Check IsOperationPending to avoid marking a volume as detached if it's in the process of mounting.
+		if !rc.desiredStateOfWorld.VolumeExists(attachedVolume.VolumeName, attachedVolume.SELinuxMountContext) &&
+			!rc.operationExecutor.IsOperationPending(attachedVolume.VolumeName, nestedpendingoperations.EmptyUniquePodName, nestedpendingoperations.EmptyNodeName) {
+			if attachedVolume.DeviceMayBeMounted() {
+				// Volume is globally mounted to device, unmount it
+				klog.V(5).InfoS(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", ""))
+				err := rc.operationExecutor.UnmountDevice(
+					attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.hostutil)
+				if err != nil && !isExpectedError(err) {
+					klog.ErrorS(err, attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.UnmountDevice failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error())
+				}
+				if err == nil {
+					klog.InfoS(attachedVolume.GenerateMsgDetailed("operationExecutor.UnmountDevice started", ""))
+				}
+			} else {
+				// Volume is attached to node, detach it
+				// Kubelet not responsible for detaching or this volume has a non-attachable volume plugin.
+				if rc.controllerAttachDetachEnabled || !attachedVolume.PluginIsAttachable {
+					rc.actualStateOfWorld.MarkVolumeAsDetached(attachedVolume.VolumeName, attachedVolume.NodeName)
+					klog.InfoS(attachedVolume.GenerateMsgDetailed("Volume detached", fmt.Sprintf("DevicePath %q", attachedVolume.DevicePath)))
+				} else {
+					// Only detach if kubelet detach is enabled
+					klog.V(5).InfoS(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.DetachVolume", ""))
+					err := rc.operationExecutor.DetachVolume(
+						attachedVolume.AttachedVolume, false /* verifySafeToDetach */, rc.actualStateOfWorld)
+					if err != nil && !isExpectedError(err) {
+						klog.ErrorS(err, attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.DetachVolume failed (controllerAttachDetachEnabled %v)", rc.controllerAttachDetachEnabled), err).Error())
+					}
+					if err == nil {
+						klog.InfoS(attachedVolume.GenerateMsgDetailed("operationExecutor.DetachVolume started", ""))
+					}
+				}
+			}
+		}
+	}
+}
+
+// ignore nestedpendingoperations.IsAlreadyExists and exponentialbackoff.IsExponentialBackoff errors, they are expected.
+func isExpectedError(err error) bool {
+	return nestedpendingoperations.IsAlreadyExists(err) || exponentialbackoff.IsExponentialBackoff(err) || operationexecutor.IsMountFailedPreconditionError(err)
+}

pkg/kubelet/volumemanager/reconciler/reconciler_new.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reconciler
+
+import (
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog/v2"
+)
+
+// TODO: move to reconciler.go and remove old code there when SELinuxMountReadWriteOncePod is GA
+
+// TODO: Replace Run() when SELinuxMountReadWriteOncePod is GA
+func (rc *reconciler) runNew(stopCh <-chan struct{}) {
+	rc.reconstructVolumes()
+	klog.InfoS("Reconciler: start to sync state")
+	wait.Until(rc.reconcileNew, rc.loopSleepDuration, stopCh)
+}
+
+func (rc *reconciler) reconcileNew() {
+	readyToUnmount := rc.readyToUnmount()
+	if readyToUnmount {
+		// Unmounts are triggered before mounts so that a volume that was
+		// referenced by a pod that was deleted and is now referenced by another
+		// pod is unmounted from the first pod before being mounted to the new
+		// pod.
+		rc.unmountVolumes()
+	}
+
+	// Next we mount required volumes. This function could also trigger
+	// attach if kubelet is responsible for attaching volumes.
+	// If underlying PVC was resized while in-use then this function also handles volume
+	// resizing.
+	rc.mountOrAttachVolumes()
+
+	// Unmount volumes only when DSW and ASW are fully populated to prevent unmounting a volume
+	// that is still needed, but it did not reach DSW yet.
+	if readyToUnmount {
+		// Ensure devices that should be detached/unmounted are detached/unmounted.
+		rc.unmountDetachDevices()
+
+		// Clean up any orphan volumes that failed reconstruction.
+		rc.cleanOrphanVolumes()
+	}
+
+	if len(rc.volumesNeedDevicePath) != 0 {
+		rc.updateReconstructedDevicePaths()
+	}
+
+	if len(rc.volumesNeedReportedInUse) != 0 && rc.populatorHasAddedPods() {
+		// Once DSW is populated, mark all reconstructed as reported in node.status,
+		// so they can proceed with MountDevice / SetUp.
+		rc.desiredStateOfWorld.MarkVolumesReportedInUse(rc.volumesNeedReportedInUse)
+		rc.volumesNeedReportedInUse = nil
+	}
+}

pkg/kubelet/volumemanager/reconciler/reconstruct.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reconciler
+
+import (
+	"context"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations"
+	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
+)
+
+// sync process tries to observe the real world by scanning all pods' volume directories from the disk.
+// If the actual and desired state of worlds are not consistent with the observed world, it means that some
+// mounted volumes are left out probably during kubelet restart. This process will reconstruct
+// the volumes and update the actual and desired states. For the volumes that cannot support reconstruction,
+// it will try to clean up the mount paths with operation executor.
+func (rc *reconciler) sync() {
+	defer rc.updateLastSyncTime()
+	rc.syncStates(rc.kubeletPodsDir)
+}
+
+// syncStates scans the volume directories under the given pod directory.
+// If the volume is not in desired state of world, this function will reconstruct
+// the volume related information and put it in both the actual and desired state of worlds.
+// For some volume plugins that cannot support reconstruction, it will clean up the existing
+// mount points since the volume is no long needed (removed from desired state)
+func (rc *reconciler) syncStates(kubeletPodDir string) {
+	// Get volumes information by reading the pod's directory
+	podVolumes, err := getVolumesFromPodDir(kubeletPodDir)
+	if err != nil {
+		klog.ErrorS(err, "Cannot get volumes from disk, skip sync states for volume reconstruction")
+		return
+	}
+	volumesNeedUpdate := make(map[v1.UniqueVolumeName]*globalVolumeInfo)
+	volumeNeedReport := []v1.UniqueVolumeName{}
+	for _, volume := range podVolumes {
+		if rc.actualStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
+			klog.V(4).InfoS("Volume exists in actual state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+			// There is nothing to reconstruct
+			continue
+		}
+		volumeInDSW := rc.desiredStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName)
+
+		reconstructedVolume, err := rc.reconstructVolume(volume)
+		if err != nil {
+			if volumeInDSW {
+				// Some pod needs the volume, don't clean it up and hope that
+				// reconcile() calls SetUp and reconstructs the volume in ASW.
+				klog.V(4).InfoS("Volume exists in desired state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+				continue
+			}
+			// No pod needs the volume.
+			klog.InfoS("Could not construct volume information, cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName, "err", err)
+			rc.cleanupMounts(volume)
+			continue
+		}
+		gvl := &globalVolumeInfo{
+			volumeName:        reconstructedVolume.volumeName,
+			volumeSpec:        reconstructedVolume.volumeSpec,
+			devicePath:        reconstructedVolume.devicePath,
+			deviceMounter:     reconstructedVolume.deviceMounter,
+			blockVolumeMapper: reconstructedVolume.blockVolumeMapper,
+			mounter:           reconstructedVolume.mounter,
+		}
+		if cachedInfo, ok := volumesNeedUpdate[reconstructedVolume.volumeName]; ok {
+			gvl = cachedInfo
+		}
+		gvl.addPodVolume(reconstructedVolume)
+		if volumeInDSW {
+			// Some pod needs the volume. And it exists on disk. Some previous
+			// kubelet must have created the directory, therefore it must have
+			// reported the volume as in use. Mark the volume as in use also in
+			// this new kubelet so reconcile() calls SetUp and re-mounts the
+			// volume if it's necessary.
+			volumeNeedReport = append(volumeNeedReport, reconstructedVolume.volumeName)
+			rc.skippedDuringReconstruction[reconstructedVolume.volumeName] = gvl
+			klog.V(4).InfoS("Volume exists in desired state, marking as InUse", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+			continue
+		}
+		// There is no pod that uses the volume.
+		if rc.operationExecutor.IsOperationPending(reconstructedVolume.volumeName, nestedpendingoperations.EmptyUniquePodName, nestedpendingoperations.EmptyNodeName) {
+			klog.InfoS("Volume is in pending operation, skip cleaning up mounts")
+		}
+		klog.V(2).InfoS("Reconciler sync states: could not find pod information in desired state, update it in actual state", "reconstructedVolume", reconstructedVolume)
+		volumesNeedUpdate[reconstructedVolume.volumeName] = gvl
+	}
+
+	if len(volumesNeedUpdate) > 0 {
+		if err = rc.updateStates(volumesNeedUpdate); err != nil {
+			klog.ErrorS(err, "Error occurred during reconstruct volume from disk")
+		}
+	}
+	if len(volumeNeedReport) > 0 {
+		rc.desiredStateOfWorld.MarkVolumesReportedInUse(volumeNeedReport)
+	}
+}
+
+// updateDevicePath gets the node status to retrieve volume device path information.
+func (rc *reconciler) updateDevicePath(volumesNeedUpdate map[v1.UniqueVolumeName]*globalVolumeInfo) {
+	node, fetchErr := rc.kubeClient.CoreV1().Nodes().Get(context.TODO(), string(rc.nodeName), metav1.GetOptions{})
+	if fetchErr != nil {
+		klog.ErrorS(fetchErr, "UpdateStates in reconciler: could not get node status with error")
+	} else {
+		for _, attachedVolume := range node.Status.VolumesAttached {
+			if volume, exists := volumesNeedUpdate[attachedVolume.Name]; exists {
+				volume.devicePath = attachedVolume.DevicePath
+				volumesNeedUpdate[attachedVolume.Name] = volume
+				klog.V(4).InfoS("Update devicePath from node status for volume", "volumeName", attachedVolume.Name, "path", volume.devicePath)
+			}
+		}
+	}
+}
+
+func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*globalVolumeInfo) error {
+	// Get the node status to retrieve volume device path information.
+	// Skip reporting devicePath in node objects if kubeClient is nil.
+	// In standalone mode, kubelet is not expected to mount any attachable volume types or secret, configmaps etc.
+	if rc.kubeClient != nil {
+		rc.updateDevicePath(volumesNeedUpdate)
+	}
+
+	for _, gvl := range volumesNeedUpdate {
+		err := rc.actualStateOfWorld.MarkVolumeAsAttached(
+			//TODO: the devicePath might not be correct for some volume plugins: see issue #54108
+			gvl.volumeName, gvl.volumeSpec, rc.nodeName, gvl.devicePath)
+		if err != nil {
+			klog.ErrorS(err, "Could not add volume information to actual state of world", "volumeName", gvl.volumeName)
+			continue
+		}
+		for _, volume := range gvl.podVolumes {
+			err = rc.markVolumeState(volume, operationexecutor.VolumeMounted)
+			if err != nil {
+				klog.ErrorS(err, "Could not add pod to volume information to actual state of world", "pod", klog.KObj(volume.pod))
+				continue
+			}
+			klog.V(2).InfoS("Volume is marked as mounted and added into the actual state", "pod", klog.KObj(volume.pod), "podName", volume.podName, "volumeName", volume.volumeName)
+		}
+		// If the volume has device to mount, we mark its device as mounted.
+		if gvl.deviceMounter != nil || gvl.blockVolumeMapper != nil {
+			deviceMountPath, err := getDeviceMountPath(gvl)
+			if err != nil {
+				klog.ErrorS(err, "Could not find device mount path for volume", "volumeName", gvl.volumeName)
+				continue
+			}
+			// TODO(jsafrane): add reconstructed SELinux context
+			err = rc.actualStateOfWorld.MarkDeviceAsMounted(gvl.volumeName, gvl.devicePath, deviceMountPath, "")
+			if err != nil {
+				klog.ErrorS(err, "Could not mark device is mounted to actual state of world", "volume", gvl.volumeName)
+				continue
+			}
+			klog.V(2).InfoS("Volume is marked device as mounted and added into the actual state", "volumeName", gvl.volumeName)
+		}
+	}
+	return nil
+}
+
+func (rc *reconciler) markVolumeState(volume *reconstructedVolume, volumeState operationexecutor.VolumeMountState) error {
+	markVolumeOpts := operationexecutor.MarkVolumeOpts{
+		PodName:             volume.podName,
+		PodUID:              types.UID(volume.podName),
+		VolumeName:          volume.volumeName,
+		Mounter:             volume.mounter,
+		BlockVolumeMapper:   volume.blockVolumeMapper,
+		OuterVolumeSpecName: volume.outerVolumeSpecName,
+		VolumeGidVolume:     volume.volumeGidValue,
+		VolumeSpec:          volume.volumeSpec,
+		VolumeMountState:    volumeState,
+	}
+	err := rc.actualStateOfWorld.MarkVolumeAsMounted(markVolumeOpts)
+	return err
+}

pkg/kubelet/volumemanager/reconciler/reconstruct_common.go

@@ -0,0 +1,320 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reconciler
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/kubelet/config"
+	volumepkg "k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
+	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
+	utilpath "k8s.io/utils/path"
+	utilstrings "k8s.io/utils/strings"
+)
+
+type podVolume struct {
+	podName        volumetypes.UniquePodName
+	volumeSpecName string
+	volumePath     string
+	pluginName     string
+	volumeMode     v1.PersistentVolumeMode
+}
+
+type reconstructedVolume struct {
+	volumeName          v1.UniqueVolumeName
+	podName             volumetypes.UniquePodName
+	volumeSpec          *volumepkg.Spec
+	outerVolumeSpecName string
+	pod                 *v1.Pod
+	volumeGidValue      string
+	devicePath          string
+	mounter             volumepkg.Mounter
+	deviceMounter       volumepkg.DeviceMounter
+	blockVolumeMapper   volumepkg.BlockVolumeMapper
+}
+
+// globalVolumeInfo stores reconstructed volume information
+// for each pod that was using that volume.
+type globalVolumeInfo struct {
+	volumeName        v1.UniqueVolumeName
+	volumeSpec        *volumepkg.Spec
+	devicePath        string
+	mounter           volumepkg.Mounter
+	deviceMounter     volumepkg.DeviceMounter
+	blockVolumeMapper volumepkg.BlockVolumeMapper
+	podVolumes        map[volumetypes.UniquePodName]*reconstructedVolume
+}
+
+func (rc *reconciler) updateLastSyncTime() {
+	rc.timeOfLastSync = time.Now()
+}
+
+func (rc *reconciler) StatesHasBeenSynced() bool {
+	return !rc.timeOfLastSync.IsZero()
+}
+
+func (gvi *globalVolumeInfo) addPodVolume(rcv *reconstructedVolume) {
+	if gvi.podVolumes == nil {
+		gvi.podVolumes = map[volumetypes.UniquePodName]*reconstructedVolume{}
+	}
+	gvi.podVolumes[rcv.podName] = rcv
+}
+
+func (rc *reconciler) cleanupMounts(volume podVolume) {
+	klog.V(2).InfoS("Reconciler sync states: could not find volume information in desired state, clean up the mount points", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+	mountedVolume := operationexecutor.MountedVolume{
+		PodName:             volume.podName,
+		VolumeName:          v1.UniqueVolumeName(volume.volumeSpecName),
+		InnerVolumeSpecName: volume.volumeSpecName,
+		PluginName:          volume.pluginName,
+		PodUID:              types.UID(volume.podName),
+	}
+	// TODO: Currently cleanupMounts only includes UnmountVolume operation. In the next PR, we will add
+	// to unmount both volume and device in the same routine.
+	err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
+	if err != nil {
+		klog.ErrorS(err, mountedVolume.GenerateErrorDetailed("volumeHandler.UnmountVolumeHandler for UnmountVolume failed", err).Error())
+		return
+	}
+}
+
+// getDeviceMountPath returns device mount path for block volume which
+// implements BlockVolumeMapper or filesystem volume which implements
+// DeviceMounter
+func getDeviceMountPath(gvi *globalVolumeInfo) (string, error) {
+	if gvi.blockVolumeMapper != nil {
+		// for block gvi, we return its global map path
+		return gvi.blockVolumeMapper.GetGlobalMapPath(gvi.volumeSpec)
+	} else if gvi.deviceMounter != nil {
+		// for filesystem gvi, we return its device mount path if the plugin implements DeviceMounter
+		return gvi.deviceMounter.GetDeviceMountPath(gvi.volumeSpec)
+	} else {
+		return "", fmt.Errorf("blockVolumeMapper or deviceMounter required")
+	}
+}
+
+// getVolumesFromPodDir scans through the volumes directories under the given pod directory.
+// It returns a list of pod volume information including pod's uid, volume's plugin name, mount path,
+// and volume spec name.
+func getVolumesFromPodDir(podDir string) ([]podVolume, error) {
+	podsDirInfo, err := os.ReadDir(podDir)
+	if err != nil {
+		return nil, err
+	}
+	volumes := []podVolume{}
+	for i := range podsDirInfo {
+		if !podsDirInfo[i].IsDir() {
+			continue
+		}
+		podName := podsDirInfo[i].Name()
+		podDir := path.Join(podDir, podName)
+
+		// Find filesystem volume information
+		// ex. filesystem volume: /pods/{podUid}/volume/{escapeQualifiedPluginName}/{volumeName}
+		volumesDirs := map[v1.PersistentVolumeMode]string{
+			v1.PersistentVolumeFilesystem: path.Join(podDir, config.DefaultKubeletVolumesDirName),
+		}
+		// Find block volume information
+		// ex. block volume: /pods/{podUid}/volumeDevices/{escapeQualifiedPluginName}/{volumeName}
+		volumesDirs[v1.PersistentVolumeBlock] = path.Join(podDir, config.DefaultKubeletVolumeDevicesDirName)
+
+		for volumeMode, volumesDir := range volumesDirs {
+			var volumesDirInfo []fs.DirEntry
+			if volumesDirInfo, err = os.ReadDir(volumesDir); err != nil {
+				// Just skip the loop because given volumesDir doesn't exist depending on volumeMode
+				continue
+			}
+			for _, volumeDir := range volumesDirInfo {
+				pluginName := volumeDir.Name()
+				volumePluginPath := path.Join(volumesDir, pluginName)
+				volumePluginDirs, err := utilpath.ReadDirNoStat(volumePluginPath)
+				if err != nil {
+					klog.ErrorS(err, "Could not read volume plugin directory", "volumePluginPath", volumePluginPath)
+					continue
+				}
+				unescapePluginName := utilstrings.UnescapeQualifiedName(pluginName)
+				for _, volumeName := range volumePluginDirs {
+					volumePath := path.Join(volumePluginPath, volumeName)
+					klog.V(5).InfoS("Volume path from volume plugin directory", "podName", podName, "volumePath", volumePath)
+					volumes = append(volumes, podVolume{
+						podName:        volumetypes.UniquePodName(podName),
+						volumeSpecName: volumeName,
+						volumePath:     volumePath,
+						pluginName:     unescapePluginName,
+						volumeMode:     volumeMode,
+					})
+				}
+			}
+		}
+	}
+	klog.V(4).InfoS("Get volumes from pod directory", "path", podDir, "volumes", volumes)
+	return volumes, nil
+}
+
+// Reconstruct volume data structure by reading the pod's volume directories
+func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, error) {
+	// plugin initializations
+	plugin, err := rc.volumePluginMgr.FindPluginByName(volume.pluginName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create pod object
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID: types.UID(volume.podName),
+		},
+	}
+	mapperPlugin, err := rc.volumePluginMgr.FindMapperPluginByName(volume.pluginName)
+	if err != nil {
+		return nil, err
+	}
+	if volume.volumeMode == v1.PersistentVolumeBlock && mapperPlugin == nil {
+		return nil, fmt.Errorf("could not find block volume plugin %q (spec.Name: %q) pod %q (UID: %q)", volume.pluginName, volume.volumeSpecName, volume.podName, pod.UID)
+	}
+
+	reconstructed, err := rc.operationExecutor.ReconstructVolumeOperation(
+		volume.volumeMode,
+		plugin,
+		mapperPlugin,
+		pod.UID,
+		volume.podName,
+		volume.volumeSpecName,
+		volume.volumePath,
+		volume.pluginName)
+	if err != nil {
+		return nil, err
+	}
+	volumeSpec := reconstructed.Spec
+
+	// We have to find the plugins by volume spec (NOT by plugin name) here
+	// in order to correctly reconstruct ephemeral volume types.
+	// Searching by spec checks whether the volume is actually attachable
+	// (i.e. has a PV) whereas searching by plugin name can only tell whether
+	// the plugin supports attachable volumes.
+	attachablePlugin, err := rc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec)
+	if err != nil {
+		return nil, err
+	}
+	deviceMountablePlugin, err := rc.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	var uniqueVolumeName v1.UniqueVolumeName
+	if attachablePlugin != nil || deviceMountablePlugin != nil {
+		uniqueVolumeName, err = util.GetUniqueVolumeNameFromSpec(plugin, volumeSpec)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		uniqueVolumeName = util.GetUniqueVolumeNameFromSpecWithPod(volume.podName, plugin, volumeSpec)
+	}
+
+	var volumeMapper volumepkg.BlockVolumeMapper
+	var volumeMounter volumepkg.Mounter
+	var deviceMounter volumepkg.DeviceMounter
+	// Path to the mount or block device to check
+	var checkPath string
+
+	if volume.volumeMode == v1.PersistentVolumeBlock {
+		var newMapperErr error
+		volumeMapper, newMapperErr = mapperPlugin.NewBlockVolumeMapper(
+			volumeSpec,
+			pod,
+			volumepkg.VolumeOptions{})
+		if newMapperErr != nil {
+			return nil, fmt.Errorf(
+				"reconstructVolume.NewBlockVolumeMapper failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
+				uniqueVolumeName,
+				volumeSpec.Name(),
+				volume.podName,
+				pod.UID,
+				newMapperErr)
+		}
+		mapDir, linkName := volumeMapper.GetPodDeviceMapPath()
+		checkPath = filepath.Join(mapDir, linkName)
+	} else {
+		var err error
+		volumeMounter, err = plugin.NewMounter(
+			volumeSpec,
+			pod,
+			volumepkg.VolumeOptions{})
+		if err != nil {
+			return nil, fmt.Errorf(
+				"reconstructVolume.NewMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
+				uniqueVolumeName,
+				volumeSpec.Name(),
+				volume.podName,
+				pod.UID,
+				err)
+		}
+		checkPath = volumeMounter.GetPath()
+		if deviceMountablePlugin != nil {
+			deviceMounter, err = deviceMountablePlugin.NewDeviceMounter()
+			if err != nil {
+				return nil, fmt.Errorf("reconstructVolume.NewDeviceMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
+					uniqueVolumeName,
+					volumeSpec.Name(),
+					volume.podName,
+					pod.UID,
+					err)
+			}
+		}
+	}
+
+	// Check existence of mount point for filesystem volume or symbolic link for block volume
+	isExist, checkErr := rc.operationExecutor.CheckVolumeExistenceOperation(volumeSpec, checkPath, volumeSpec.Name(), rc.mounter, uniqueVolumeName, volume.podName, pod.UID, attachablePlugin)
+	if checkErr != nil {
+		return nil, checkErr
+	}
+	// If mount or symlink doesn't exist, volume reconstruction should be failed
+	if !isExist {
+		return nil, fmt.Errorf("volume: %q is not mounted", uniqueVolumeName)
+	}
+
+	reconstructedVolume := &reconstructedVolume{
+		volumeName: uniqueVolumeName,
+		podName:    volume.podName,
+		volumeSpec: volumeSpec,
+		// volume.volumeSpecName is actually InnerVolumeSpecName. It will not be used
+		// for volume cleanup.
+		// in case pod is added back to desired state, outerVolumeSpecName will be updated from dsw information.
+		// See issue #103143 and its fix for details.
+		outerVolumeSpecName: volume.volumeSpecName,
+		pod:                 pod,
+		deviceMounter:       deviceMounter,
+		volumeGidValue:      "",
+		// devicePath is updated during updateStates() by checking node status's VolumesAttached data.
+		// TODO: get device path directly from the volume mount path.
+		devicePath:        "",
+		mounter:           volumeMounter,
+		blockVolumeMapper: volumeMapper,
+	}
+	return reconstructedVolume, nil
+}

pkg/kubelet/volumemanager/reconciler/reconstruct_new.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reconciler
+
+import (
+	"context"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
+)
+
+// TODO: move to reconstruct.go and remove old code there.
+
+// readyToUnmount returns true when reconciler can start unmounting volumes.
+func (rc *reconciler) readyToUnmount() bool {
+	// During kubelet startup, all volumes present on disk are added as uncertain to ASW.
+	// Allow unmount only when DSW is fully populated to prevent unmounting volumes that
+	// did not reach DSW yet.
+	if !rc.populatorHasAddedPods() {
+		return false
+	}
+
+	// Allow unmount only when ASW device paths were corrected from node.status to prevent
+	// calling unmount with a wrong devicePath.
+	if len(rc.volumesNeedDevicePath) != 0 {
+		return false
+	}
+	return true
+}
+
+// reconstructVolumes tries to reconstruct the actual state of world by scanning all pods' volume
+// directories from the disk. For the volumes that cannot support or fail reconstruction, it will
+// put the volumes to volumesFailedReconstruction to be cleaned up later when DesiredStateOfWorld
+// is populated.
+func (rc *reconciler) reconstructVolumes() {
+	defer rc.updateLastSyncTime()
+	// Get volumes information by reading the pod's directory
+	podVolumes, err := getVolumesFromPodDir(rc.kubeletPodsDir)
+	if err != nil {
+		klog.ErrorS(err, "Cannot get volumes from disk, skip sync states for volume reconstruction")
+		return
+	}
+	reconstructedVolumes := make(map[v1.UniqueVolumeName]*globalVolumeInfo)
+	reconstructedVolumeNames := []v1.UniqueVolumeName{}
+	for _, volume := range podVolumes {
+		if rc.actualStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
+			klog.V(4).InfoS("Volume exists in actual state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+			// There is nothing to reconstruct
+			continue
+		}
+		reconstructedVolume, err := rc.reconstructVolume(volume)
+		if err != nil {
+			klog.InfoS("Could not construct volume information", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName, "err", err)
+			// We can't reconstruct the volume. Remember to check DSW after it's fully populated and force unmount the volume when it's orphaned.
+			rc.volumesFailedReconstruction = append(rc.volumesFailedReconstruction, volume)
+			continue
+		}
+		klog.V(4).InfoS("Adding reconstructed volume to actual state and node status", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+		gvl := &globalVolumeInfo{
+			volumeName:        reconstructedVolume.volumeName,
+			volumeSpec:        reconstructedVolume.volumeSpec,
+			devicePath:        reconstructedVolume.devicePath,
+			deviceMounter:     reconstructedVolume.deviceMounter,
+			blockVolumeMapper: reconstructedVolume.blockVolumeMapper,
+			mounter:           reconstructedVolume.mounter,
+		}
+		if cachedInfo, ok := reconstructedVolumes[reconstructedVolume.volumeName]; ok {
+			gvl = cachedInfo
+		}
+		gvl.addPodVolume(reconstructedVolume)
+
+		reconstructedVolumeNames = append(reconstructedVolumeNames, reconstructedVolume.volumeName)
+		reconstructedVolumes[reconstructedVolume.volumeName] = gvl
+	}
+
+	if len(reconstructedVolumes) > 0 {
+		// Add the volumes to ASW
+		rc.updateStatesNew(reconstructedVolumes)
+
+		// The reconstructed volumes are mounted, hence a previous kubelet must have already put it into node.status.volumesInUse.
+		// Remember to update DSW with this information.
+		rc.volumesNeedReportedInUse = reconstructedVolumeNames
+		// Remember to update devicePath from node.status.volumesAttached
+		rc.volumesNeedDevicePath = reconstructedVolumeNames
+	}
+	klog.V(2).InfoS("Volume reconstruction finished")
+}
+
+func (rc *reconciler) updateStatesNew(reconstructedVolumes map[v1.UniqueVolumeName]*globalVolumeInfo) {
+	for _, gvl := range reconstructedVolumes {
+		err := rc.actualStateOfWorld.MarkVolumeAsAttached(
+			//TODO: the devicePath might not be correct for some volume plugins: see issue #54108
+			gvl.volumeName, gvl.volumeSpec, rc.nodeName, gvl.devicePath)
+		if err != nil {
+			klog.ErrorS(err, "Could not add volume information to actual state of world", "volumeName", gvl.volumeName)
+			continue
+		}
+		for _, volume := range gvl.podVolumes {
+			markVolumeOpts := operationexecutor.MarkVolumeOpts{
+				PodName:             volume.podName,
+				PodUID:              types.UID(volume.podName),
+				VolumeName:          volume.volumeName,
+				Mounter:             volume.mounter,
+				BlockVolumeMapper:   volume.blockVolumeMapper,
+				OuterVolumeSpecName: volume.outerVolumeSpecName,
+				VolumeGidVolume:     volume.volumeGidValue,
+				VolumeSpec:          volume.volumeSpec,
+				VolumeMountState:    operationexecutor.VolumeMountUncertain,
+			}
+
+			_, err = rc.actualStateOfWorld.CheckAndMarkVolumeAsUncertainViaReconstruction(markVolumeOpts)
+			if err != nil {
+				klog.ErrorS(err, "Could not add pod to volume information to actual state of world", "pod", klog.KObj(volume.pod))
+				continue
+			}
+			klog.V(2).InfoS("Volume is marked as uncertain and added into the actual state", "pod", klog.KObj(volume.pod), "podName", volume.podName, "volumeName", volume.volumeName)
+		}
+		// If the volume has device to mount, we mark its device as uncertain.
+		if gvl.deviceMounter != nil || gvl.blockVolumeMapper != nil {
+			deviceMountPath, err := getDeviceMountPath(gvl)
+			if err != nil {
+				klog.ErrorS(err, "Could not find device mount path for volume", "volumeName", gvl.volumeName)
+				continue
+			}
+			err = rc.actualStateOfWorld.MarkDeviceAsUncertain(gvl.volumeName, gvl.devicePath, deviceMountPath, "")
+			if err != nil {
+				klog.ErrorS(err, "Could not mark device is uncertain to actual state of world", "volumeName", gvl.volumeName, "deviceMountPath", deviceMountPath)
+				continue
+			}
+			klog.V(2).InfoS("Volume is marked device as uncertain and added into the actual state", "volumeName", gvl.volumeName, "deviceMountPath", deviceMountPath)
+		}
+	}
+}
+
+// cleanOrphanVolumes tries to clean up all volumes that failed reconstruction.
+func (rc *reconciler) cleanOrphanVolumes() {
+	if len(rc.volumesFailedReconstruction) == 0 {
+		return
+	}
+
+	for _, volume := range rc.volumesFailedReconstruction {
+		if rc.desiredStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
+			// Some pod needs the volume, don't clean it up and hope that
+			// reconcile() calls SetUp and reconstructs the volume in ASW.
+			klog.V(4).InfoS("Volume exists in desired state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+			continue
+		}
+		klog.InfoS("Cleaning up mounts for volume that could not be reconstructed", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
+		rc.cleanupMounts(volume)
+	}
+
+	klog.V(2).InfoS("Orphan volume cleanup finished")
+	// Clean the cache, cleanup is one shot operation.
+	rc.volumesFailedReconstruction = make([]podVolume, 0)
+}
+
+// updateReconstructedDevicePaths tries to file devicePaths of reconstructed volumes from
+// node.Status.VolumesAttached. This can be done only after connection to the API
+// server is established, i.e. it can't be part of reconstructVolumes().
+func (rc *reconciler) updateReconstructedDevicePaths() {
+	klog.V(4).InfoS("Updating reconstructed devicePaths")
+
+	node, fetchErr := rc.kubeClient.CoreV1().Nodes().Get(context.TODO(), string(rc.nodeName), metav1.GetOptions{})
+	if fetchErr != nil {
+		// This may repeat few times per second until kubelet is able to read its own status for the first time.
+		klog.V(2).ErrorS(fetchErr, "Failed to get Node status to reconstruct device paths")
+		return
+	}
+
+	for _, volumeID := range rc.volumesNeedDevicePath {
+		for _, attachedVolume := range node.Status.VolumesAttached {
+			if volumeID != attachedVolume.Name {
+				continue
+			}
+			rc.actualStateOfWorld.UpdateReconstructedDevicePath(volumeID, attachedVolume.DevicePath)
+			klog.V(4).InfoS("Updated devicePath from node status for volume", "volumeName", attachedVolume.Name, "path", attachedVolume.DevicePath)
+		}
+	}
+	klog.V(2).InfoS("DevicePaths of reconstructed volumes updated")
+	rc.volumesNeedDevicePath = nil
+}

pkg/kubelet/volumemanager/reconciler/reconstruct_new_test.go

@@ -0,0 +1,385 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reconciler
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/volume"
+	volumetesting "k8s.io/kubernetes/pkg/volume/testing"
+	"k8s.io/kubernetes/pkg/volume/util"
+)
+
+func TestReconstructVolumes(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)()
+
+	tests := []struct {
+		name                                string
+		volumePaths                         []string
+		expectedVolumesNeedReportedInUse    []string
+		expectedVolumesNeedDevicePath       []string
+		expectedVolumesFailedReconstruction []string
+		verifyFunc                          func(rcInstance *reconciler, fakePlugin *volumetesting.FakeVolumePlugin) error
+	}{
+		{
+			name: "when two pods are using same volume and both are deleted",
+			volumePaths: []string{
+				path.Join("pod1", "volumes", "fake-plugin", "pvc-abcdef"),
+				path.Join("pod2", "volumes", "fake-plugin", "pvc-abcdef"),
+			},
+			expectedVolumesNeedReportedInUse:    []string{"fake-plugin/pvc-abcdef", "fake-plugin/pvc-abcdef"},
+			expectedVolumesNeedDevicePath:       []string{"fake-plugin/pvc-abcdef", "fake-plugin/pvc-abcdef"},
+			expectedVolumesFailedReconstruction: []string{},
+			verifyFunc: func(rcInstance *reconciler, fakePlugin *volumetesting.FakeVolumePlugin) error {
+				mountedPods := rcInstance.actualStateOfWorld.GetMountedVolumes()
+				if len(mountedPods) != 0 {
+					return fmt.Errorf("expected 0 certain pods in asw got %d", len(mountedPods))
+				}
+				allPods := rcInstance.actualStateOfWorld.GetAllMountedVolumes()
+				if len(allPods) != 2 {
+					return fmt.Errorf("expected 2 uncertain pods in asw got %d", len(allPods))
+				}
+				volumes := rcInstance.actualStateOfWorld.GetPossiblyMountedVolumesForPod("pod1")
+				if len(volumes) != 1 {
+					return fmt.Errorf("expected 1 uncertain volume in asw got %d", len(volumes))
+				}
+				// The volume should be marked as reconstructed in ASW
+				if reconstructed := rcInstance.actualStateOfWorld.IsVolumeReconstructed("fake-plugin/pvc-abcdef", "pod1"); !reconstructed {
+					t.Errorf("expected volume to be marked as reconstructed, got %v", reconstructed)
+				}
+				return nil
+			},
+		},
+		{
+			name: "when reconstruction fails for a volume, volumes should be cleaned up",
+			volumePaths: []string{
+				path.Join("pod1", "volumes", "missing-plugin", "pvc-abcdef"),
+			},
+			expectedVolumesNeedReportedInUse:    []string{},
+			expectedVolumesNeedDevicePath:       []string{},
+			expectedVolumesFailedReconstruction: []string{"pvc-abcdef"},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tmpKubeletDir, err := os.MkdirTemp("", "")
+			if err != nil {
+				t.Fatalf("can't make a temp directory for kubeletPods: %v", err)
+			}
+			defer os.RemoveAll(tmpKubeletDir)
+
+			// create kubelet pod directory
+			tmpKubeletPodDir := filepath.Join(tmpKubeletDir, "pods")
+			os.MkdirAll(tmpKubeletPodDir, 0755)
+
+			mountPaths := []string{}
+
+			// create pod and volume directories so as reconciler can find them.
+			for _, volumePath := range tc.volumePaths {
+				vp := filepath.Join(tmpKubeletPodDir, volumePath)
+				mountPaths = append(mountPaths, vp)
+				os.MkdirAll(vp, 0755)
+			}
+
+			rc, fakePlugin := getReconciler(tmpKubeletDir, t, mountPaths)
+			rcInstance, _ := rc.(*reconciler)
+
+			// Act
+			rcInstance.reconstructVolumes()
+
+			// Assert
+			// Convert to []UniqueVolumeName
+			expectedVolumes := make([]v1.UniqueVolumeName, len(tc.expectedVolumesNeedDevicePath))
+			for i := range tc.expectedVolumesNeedDevicePath {
+				expectedVolumes[i] = v1.UniqueVolumeName(tc.expectedVolumesNeedDevicePath[i])
+			}
+			if !reflect.DeepEqual(expectedVolumes, rcInstance.volumesNeedDevicePath) {
+				t.Errorf("Expected expectedVolumesNeedDevicePath:\n%v\n got:\n%v", expectedVolumes, rcInstance.volumesNeedDevicePath)
+			}
+
+			expectedVolumes = make([]v1.UniqueVolumeName, len(tc.expectedVolumesNeedReportedInUse))
+			for i := range tc.expectedVolumesNeedReportedInUse {
+				expectedVolumes[i] = v1.UniqueVolumeName(tc.expectedVolumesNeedReportedInUse[i])
+			}
+			if !reflect.DeepEqual(expectedVolumes, rcInstance.volumesNeedReportedInUse) {
+				t.Errorf("Expected volumesNeedReportedInUse:\n%v\n got:\n%v", expectedVolumes, rcInstance.volumesNeedReportedInUse)
+			}
+
+			volumesFailedReconstruction := sets.NewString()
+			for _, vol := range rcInstance.volumesFailedReconstruction {
+				volumesFailedReconstruction.Insert(vol.volumeSpecName)
+			}
+			if !reflect.DeepEqual(volumesFailedReconstruction.List(), tc.expectedVolumesFailedReconstruction) {
+				t.Errorf("Expected volumesFailedReconstruction:\n%v\n got:\n%v", tc.expectedVolumesFailedReconstruction, volumesFailedReconstruction.List())
+			}
+
+			if tc.verifyFunc != nil {
+				if err := tc.verifyFunc(rcInstance, fakePlugin); err != nil {
+					t.Errorf("Test %s failed: %v", tc.name, err)
+				}
+			}
+		})
+	}
+}
+
+func TestCleanOrphanVolumes(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)()
+
+	type podInfo struct {
+		podName         string
+		podUID          string
+		outerVolumeName string
+		innerVolumeName string
+	}
+	defaultPodInfo := podInfo{
+		podName:         "pod1",
+		podUID:          "pod1uid",
+		outerVolumeName: "volume-name",
+		innerVolumeName: "volume-name",
+	}
+	defaultVolume := podVolume{
+		podName:        "pod1uid",
+		volumeSpecName: "volume-name",
+		volumePath:     "",
+		pluginName:     "fake-plugin",
+		volumeMode:     v1.PersistentVolumeFilesystem,
+	}
+
+	tests := []struct {
+		name                        string
+		podInfos                    []podInfo
+		volumesFailedReconstruction []podVolume
+		expectedUnmounts            int
+	}{
+		{
+			name:                        "volume is in DSW and is not cleaned",
+			podInfos:                    []podInfo{defaultPodInfo},
+			volumesFailedReconstruction: []podVolume{defaultVolume},
+			expectedUnmounts:            0,
+		},
+		{
+			name:                        "volume is not in DSW and is cleaned",
+			podInfos:                    []podInfo{},
+			volumesFailedReconstruction: []podVolume{defaultVolume},
+			expectedUnmounts:            1,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			// Arrange
+			tmpKubeletDir, err := os.MkdirTemp("", "")
+			if err != nil {
+				t.Fatalf("can't make a temp directory for kubeletPods: %v", err)
+			}
+			defer os.RemoveAll(tmpKubeletDir)
+
+			// create kubelet pod directory
+			tmpKubeletPodDir := filepath.Join(tmpKubeletDir, "pods")
+			os.MkdirAll(tmpKubeletPodDir, 0755)
+
+			mountPaths := []string{}
+
+			rc, fakePlugin := getReconciler(tmpKubeletDir, t, mountPaths)
+			rcInstance, _ := rc.(*reconciler)
+			rcInstance.volumesFailedReconstruction = tc.volumesFailedReconstruction
+
+			for _, tpodInfo := range tc.podInfos {
+				pod := getInlineFakePod(tpodInfo.podName, tpodInfo.podUID, tpodInfo.outerVolumeName, tpodInfo.innerVolumeName)
+				volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]}
+				podName := util.GetUniquePodName(pod)
+				volumeName, err := rcInstance.desiredStateOfWorld.AddPodToVolume(
+					podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* SELinuxContext */)
+				if err != nil {
+					t.Fatalf("Error adding volume %s to dsow: %v", volumeSpec.Name(), err)
+				}
+				rcInstance.actualStateOfWorld.MarkVolumeAsAttached(volumeName, volumeSpec, nodeName, "")
+			}
+
+			// Act
+			rcInstance.cleanOrphanVolumes()
+
+			// Assert
+			if len(rcInstance.volumesFailedReconstruction) != 0 {
+				t.Errorf("Expected volumesFailedReconstruction to be empty, got %+v", rcInstance.volumesFailedReconstruction)
+			}
+			// Unmount runs in a go routine, wait for its finish
+			var lastErr error
+			err = retryWithExponentialBackOff(testOperationBackOffDuration, func() (bool, error) {
+				if err := verifyTearDownCalls(fakePlugin, tc.expectedUnmounts); err != nil {
+					lastErr = err
+					return false, nil
+				}
+				return true, nil
+			})
+			if err != nil {
+				t.Errorf("Error waiting for volumes to get unmounted: %s: %s", err, lastErr)
+			}
+		})
+	}
+}
+
+func verifyTearDownCalls(plugin *volumetesting.FakeVolumePlugin, expected int) error {
+	unmounters := plugin.GetUnmounters()
+	if len(unmounters) == 0 && (expected == 0) {
+		return nil
+	}
+	actualCallCount := 0
+	for _, unmounter := range unmounters {
+		actualCallCount = unmounter.GetTearDownCallCount()
+		if actualCallCount == expected {
+			return nil
+		}
+	}
+	return fmt.Errorf("expected TearDown calls %d, got %d", expected, actualCallCount)
+}
+
+func TestReconstructVolumesMount(t *testing.T) {
+	// This test checks volume reconstruction + subsequent failed mount.
+	// Since the volume is reconstructed, it must be marked as uncertain
+	// even after a final SetUp error, see https://github.com/kubernetes/kubernetes/issues/96635
+	// and https://github.com/kubernetes/kubernetes/pull/110670.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SELinuxMountReadWriteOncePod, true)()
+
+	tests := []struct {
+		name        string
+		volumePath  string
+		expectMount bool
+	}{
+		{
+			name:       "reconstructed volume is mounted",
+			volumePath: path.Join("pod1uid", "volumes", "fake-plugin", "volumename"),
+
+			expectMount: true,
+		},
+		{
+			name: "reconstructed volume fails to mount",
+			// FailOnSetupVolumeName: MountDevice succeeds, SetUp fails
+			volumePath:  path.Join("pod1uid", "volumes", "fake-plugin", volumetesting.FailOnSetupVolumeName),
+			expectMount: false,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tmpKubeletDir, err := os.MkdirTemp("", "")
+			if err != nil {
+				t.Fatalf("can't make a temp directory for kubeletPods: %v", err)
+			}
+			defer os.RemoveAll(tmpKubeletDir)
+
+			// create kubelet pod directory
+			tmpKubeletPodDir := filepath.Join(tmpKubeletDir, "pods")
+			os.MkdirAll(tmpKubeletPodDir, 0755)
+
+			// create pod and volume directories so as reconciler can find them.
+			vp := filepath.Join(tmpKubeletPodDir, tc.volumePath)
+			mountPaths := []string{vp}
+			os.MkdirAll(vp, 0755)
+
+			rc, fakePlugin := getReconciler(tmpKubeletDir, t, mountPaths)
+			rcInstance, _ := rc.(*reconciler)
+
+			// Act 1 - reconstruction
+			rcInstance.reconstructVolumes()
+
+			// Assert 1 - the volume is Uncertain
+			mountedPods := rcInstance.actualStateOfWorld.GetMountedVolumes()
+			if len(mountedPods) != 0 {
+				t.Errorf("expected 0 mounted volumes, got %+v", mountedPods)
+			}
+			allPods := rcInstance.actualStateOfWorld.GetAllMountedVolumes()
+			if len(allPods) != 1 {
+				t.Errorf("expected 1 uncertain volume in asw, got %+v", allPods)
+			}
+
+			// Arrange 2 - populate DSW
+			outerName := filepath.Base(tc.volumePath)
+			pod := getInlineFakePod("pod1", "pod1uid", outerName, outerName)
+			volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]}
+			podName := util.GetUniquePodName(pod)
+			volumeName, err := rcInstance.desiredStateOfWorld.AddPodToVolume(
+				podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */, nil /* SELinuxContext */)
+			if err != nil {
+				t.Fatalf("Error adding volume %s to dsow: %v", volumeSpec.Name(), err)
+			}
+			rcInstance.actualStateOfWorld.MarkVolumeAsAttached(volumeName, volumeSpec, nodeName, "")
+
+			rcInstance.populatorHasAddedPods = func() bool {
+				// Mark DSW populated to allow unmounting of volumes.
+				return true
+			}
+			// Mark devices paths as reconciled to allow unmounting of volumes.
+			rcInstance.volumesNeedDevicePath = nil
+
+			// Act 2 - reconcile once
+			rcInstance.reconcileNew()
+
+			// Assert 2
+			// MountDevice was attempted
+			var lastErr error
+			err = retryWithExponentialBackOff(testOperationBackOffDuration, func() (bool, error) {
+				// MountDevice should always be called and succeed
+				if err := volumetesting.VerifyMountDeviceCallCount(1, fakePlugin); err != nil {
+					lastErr = err
+					return false, nil
+				}
+				return true, nil
+			})
+			if err != nil {
+				t.Errorf("Error waiting for volumes to get mounted: %s: %s", err, lastErr)
+			}
+
+			if tc.expectMount {
+				// The volume should be fully mounted
+				waitForMount(t, fakePlugin, volumeName, rcInstance.actualStateOfWorld)
+				// SetUp was called and succeeded
+				if err := volumetesting.VerifySetUpCallCount(1, fakePlugin); err != nil {
+					t.Errorf("Expected SetUp() to be called, got %s", err)
+				}
+			} else {
+				// The test does not expect any change in ASW, yet it needs to wait for volume operations to finish
+				err = retryWithExponentialBackOff(testOperationBackOffDuration, func() (bool, error) {
+					return !rcInstance.operationExecutor.IsOperationPending(volumeName, "pod1uid", nodeName), nil
+				})
+				if err != nil {
+					t.Errorf("Error waiting for operation to get finished: %s", err)
+				}
+				// The volume is uncertain
+				mountedPods := rcInstance.actualStateOfWorld.GetMountedVolumes()
+				if len(mountedPods) != 0 {
+					t.Errorf("expected 0 mounted volumes after reconcile, got %+v", mountedPods)
+				}
+				allPods := rcInstance.actualStateOfWorld.GetAllMountedVolumes()
+				if len(allPods) != 1 {
+					t.Errorf("expected 1 mounted or uncertain volumes after reconcile, got %+v", allPods)
+				}
+			}
+
+			// Unmount was *not* attempted in any case
+			verifyTearDownCalls(fakePlugin, 0)
+		})
+	}
+}

pkg/volume/awsebs/aws_ebs.go

@@ -249,25 +249,27 @@ func getVolumeSource(
 	return nil, false, fmt.Errorf("Spec does not reference an AWS EBS volume type")
 }
 
-func (plugin *awsElasticBlockStorePlugin) ConstructVolumeSpec(volName, mountPath string) (*volume.Spec, error) {
+func (plugin *awsElasticBlockStorePlugin) ConstructVolumeSpec(volName, mountPath string) (volume.ReconstructedVolume, error) {
 	mounter := plugin.host.GetMounter(plugin.GetPluginName())
 	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
 	if !ok {
-		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
+		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
 	}
 	hu := kvh.GetHostUtil()
 	pluginMntDir := util.GetPluginMountDir(plugin.host, plugin.GetPluginName())
 	volumeID, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	volumeID, err = formatVolumeID(volumeID)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get AWS volume id from mount path %q: %v", mountPath, err)
+		return volume.ReconstructedVolume{}, fmt.Errorf("failed to get AWS volume id from mount path %q: %v", mountPath, err)
 	}
 
 	file := v1.PersistentVolumeFilesystem
-	return newAWSVolumeSpec(volName, volumeID, file), nil
+	return volume.ReconstructedVolume{
+		Spec: newAWSVolumeSpec(volName, volumeID, file),
+	}, nil
 }
 
 func (plugin *awsElasticBlockStorePlugin) RequiresFSResize() bool {

pkg/volume/azure_file/azure_file.go

@@ -202,7 +202,7 @@ func (plugin *azureFilePlugin) ExpandVolumeDevice(
 	return newSize, nil
 }
 
-func (plugin *azureFilePlugin) ConstructVolumeSpec(volName, mountPath string) (*volume.Spec, error) {
+func (plugin *azureFilePlugin) ConstructVolumeSpec(volName, mountPath string) (volume.ReconstructedVolume, error) {
 	azureVolume := &v1.Volume{
 		Name: volName,
 		VolumeSource: v1.VolumeSource{
@@ -212,7 +212,9 @@ func (plugin *azureFilePlugin) ConstructVolumeSpec(volName, mountPath string) (*
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(azureVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(azureVolume),
+	}, nil
 }
 
 // azureFile volumes represent mount of an AzureFile share.

pkg/volume/azuredd/azure_dd.go

@@ -316,18 +316,18 @@ func (plugin *azureDataDiskPlugin) NodeExpand(resizeOptions volume.NodeResizeOpt
 
 var _ volume.NodeExpandableVolumePlugin = &azureDataDiskPlugin{}
 
-func (plugin *azureDataDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *azureDataDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	mounter := plugin.host.GetMounter(plugin.GetPluginName())
 	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
 	if !ok {
-		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
+		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
 	}
 	hu := kvh.GetHostUtil()
 	pluginMntDir := util.GetPluginMountDir(plugin.host, plugin.GetPluginName())
 	sourceName, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
 
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 
 	azureVolume := &v1.Volume{
@@ -338,7 +338,9 @@ func (plugin *azureDataDiskPlugin) ConstructVolumeSpec(volumeName, mountPath str
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(azureVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(azureVolume),
+	}, nil
 }
 
 func (plugin *azureDataDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {

pkg/volume/cephfs/cephfs.go

@@ -173,7 +173,7 @@ func (plugin *cephfsPlugin) newUnmounterInternal(volName string, podUID types.UI
 	}, nil
 }
 
-func (plugin *cephfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *cephfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	cephfsVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
@@ -183,7 +183,9 @@ func (plugin *cephfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(cephfsVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(cephfsVolume),
+	}, nil
 }
 
 // CephFS volumes represent a bare host file or directory mount of an CephFS export.

pkg/volume/cephfs/cephfs_test.go

@@ -128,13 +128,13 @@ func TestConstructVolumeSpec(t *testing.T) {
 		t.Errorf("can't find cephfs plugin by name")
 	}
 
-	cephfsSpec, err := plug.(*cephfsPlugin).ConstructVolumeSpec("cephfsVolume", "/cephfsVolume/")
+	cephfsVol, err := plug.(*cephfsPlugin).ConstructVolumeSpec("cephfsVolume", "/cephfsVolume/")
 	if err != nil {
 		t.Errorf("ConstructVolumeSpec() failed: %v", err)
 	}
 
-	if cephfsSpec.Name() != "cephfsVolume" {
+	if cephfsVol.Spec.Name() != "cephfsVolume" {
-		t.Errorf("Get wrong cephfs spec name, got: %s", cephfsSpec.Name())
+		t.Errorf("Get wrong cephfs spec name, got: %s", cephfsVol.Spec.Name())
 	}
 }
 

pkg/volume/configmap/configmap.go

@@ -122,14 +122,16 @@ func (plugin *configMapPlugin) NewUnmounter(volName string, podUID types.UID) (v
 	}, nil
 }
 
-func (plugin *configMapPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *configMapPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	configMapVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
 			ConfigMap: &v1.ConfigMapVolumeSource{},
 		},
 	}
-	return volume.NewSpecFromVolume(configMapVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(configMapVolume),
+	}, nil
 }
 
 type configMapVolume struct {

pkg/volume/csi/csi_plugin.go

@@ -448,12 +448,12 @@ func (p *csiPlugin) NewUnmounter(specName string, podUID types.UID) (volume.Unmo
 	return unmounter, nil
 }
 
-func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	klog.V(4).Info(log("plugin.ConstructVolumeSpec [pv.Name=%v, path=%v]", volumeName, mountPath))
 
 	volData, err := loadVolumeData(mountPath, volDataFileName)
 	if err != nil {
-		return nil, errors.New(log("plugin.ConstructVolumeSpec failed loading volume data using [%s]: %v", mountPath, err))
+		return volume.ReconstructedVolume{}, errors.New(log("plugin.ConstructVolumeSpec failed loading volume data using [%s]: %v", mountPath, err))
 	}
 
 	klog.V(4).Info(log("plugin.ConstructVolumeSpec extracted [%#v]", volData))
@@ -464,11 +464,13 @@ func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.S
 	// use constructPVSourceSpec to construct volume construct pv source spec.
 	if storage.VolumeLifecycleMode(volData[volDataKey.volumeLifecycleMode]) == storage.VolumeLifecycleEphemeral {
 		spec = p.constructVolSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName])
-		return spec, nil
+		return volume.ReconstructedVolume{Spec: spec}, nil
 	}
-	spec = p.constructPVSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName], volData[volDataKey.volHandle])
 
-	return spec, nil
+	spec = p.constructPVSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName], volData[volDataKey.volHandle])
+	return volume.ReconstructedVolume{
+		Spec: spec,
+	}, nil
 }
 
 // constructVolSourceSpec constructs volume.Spec with CSIVolumeSource

pkg/volume/csi/csi_plugin_test.go

@@ -362,38 +362,38 @@ func TestPluginConstructVolumeSpec(t *testing.T) {
 			}
 
 			// rebuild spec
-			spec, err := plug.ConstructVolumeSpec("test-pv", filepath.Dir(csiMounter.GetPath()))
+			rec, err := plug.ConstructVolumeSpec("test-pv", filepath.Dir(csiMounter.GetPath()))
 			if err != nil {
 				t.Fatal(err)
 			}
-			if spec == nil {
+			if rec.Spec == nil {
 				t.Fatal("nil volume.Spec constructed")
 			}
 
 			// inspect spec
-			if spec.PersistentVolume == nil || spec.PersistentVolume.Spec.CSI == nil {
+			if rec.Spec.PersistentVolume == nil || rec.Spec.PersistentVolume.Spec.CSI == nil {
 				t.Fatal("CSIPersistentVolume not found in constructed spec ")
 			}
 
-			volHandle := spec.PersistentVolume.Spec.CSI.VolumeHandle
+			volHandle := rec.Spec.PersistentVolume.Spec.CSI.VolumeHandle
 			if volHandle != tc.originSpec.PersistentVolume.Spec.CSI.VolumeHandle {
 				t.Error("unexpected volumeHandle constructed:", volHandle)
 			}
-			driverName := spec.PersistentVolume.Spec.CSI.Driver
+			driverName := rec.Spec.PersistentVolume.Spec.CSI.Driver
 			if driverName != tc.originSpec.PersistentVolume.Spec.CSI.Driver {
 				t.Error("unexpected driverName constructed:", driverName)
 			}
 
-			if spec.PersistentVolume.Spec.VolumeMode == nil {
+			if rec.Spec.PersistentVolume.Spec.VolumeMode == nil {
 				t.Fatalf("Volume mode has not been set.")
 			}
 
-			if *spec.PersistentVolume.Spec.VolumeMode != api.PersistentVolumeFilesystem {
+			if *rec.Spec.PersistentVolume.Spec.VolumeMode != api.PersistentVolumeFilesystem {
-				t.Errorf("Unexpected volume mode %q", *spec.PersistentVolume.Spec.VolumeMode)
+				t.Errorf("Unexpected volume mode %q", *rec.Spec.PersistentVolume.Spec.VolumeMode)
 			}
 
-			if spec.Name() != tc.specVolID {
+			if rec.Spec.Name() != tc.specVolID {
-				t.Errorf("Unexpected spec name constructed %s", spec.Name())
+				t.Errorf("Unexpected spec name constructed %s", rec.Spec.Name())
 			}
 		})
 	}
@@ -496,44 +496,44 @@ func TestPluginConstructVolumeSpecWithInline(t *testing.T) {
 			}
 
 			// rebuild spec
-			spec, err := plug.ConstructVolumeSpec("test-pv", filepath.Dir(csiMounter.GetPath()))
+			rec, err := plug.ConstructVolumeSpec("test-pv", filepath.Dir(csiMounter.GetPath()))
 			if err != nil {
 				t.Fatal(err)
 			}
-			if spec == nil {
+			if rec.Spec == nil {
 				t.Fatal("nil volume.Spec constructed")
 			}
 
-			if spec.Name() != tc.specVolID {
+			if rec.Spec.Name() != tc.specVolID {
-				t.Errorf("unexpected spec name constructed volume.Spec: %s", spec.Name())
+				t.Errorf("unexpected spec name constructed volume.Spec: %s", rec.Spec.Name())
 			}
 
 			switch {
-			case spec.Volume != nil:
+			case rec.Spec.Volume != nil:
-				if spec.Volume.CSI == nil {
+				if rec.Spec.Volume.CSI == nil {
 					t.Error("missing CSIVolumeSource in constructed volume.Spec")
 				}
-				if spec.Volume.CSI.Driver != tc.originSpec.Volume.CSI.Driver {
+				if rec.Spec.Volume.CSI.Driver != tc.originSpec.Volume.CSI.Driver {
-					t.Error("unexpected driver in constructed volume source:", spec.Volume.CSI.Driver)
+					t.Error("unexpected driver in constructed volume source:", rec.Spec.Volume.CSI.Driver)
 				}
 
-			case spec.PersistentVolume != nil:
+			case rec.Spec.PersistentVolume != nil:
-				if spec.PersistentVolume.Spec.CSI == nil {
+				if rec.Spec.PersistentVolume.Spec.CSI == nil {
 					t.Fatal("missing CSIPersistentVolumeSource in constructed volume.spec")
 				}
-				volHandle := spec.PersistentVolume.Spec.CSI.VolumeHandle
+				volHandle := rec.Spec.PersistentVolume.Spec.CSI.VolumeHandle
 				if volHandle != tc.originSpec.PersistentVolume.Spec.CSI.VolumeHandle {
 					t.Error("unexpected volumeHandle constructed in persistent volume source:", volHandle)
 				}
-				driverName := spec.PersistentVolume.Spec.CSI.Driver
+				driverName := rec.Spec.PersistentVolume.Spec.CSI.Driver
 				if driverName != tc.originSpec.PersistentVolume.Spec.CSI.Driver {
 					t.Error("unexpected driverName constructed in persistent volume source:", driverName)
 				}
-				if spec.PersistentVolume.Spec.VolumeMode == nil {
+				if rec.Spec.PersistentVolume.Spec.VolumeMode == nil {
 					t.Fatalf("Volume mode has not been set.")
 				}
-				if *spec.PersistentVolume.Spec.VolumeMode != api.PersistentVolumeFilesystem {
+				if *rec.Spec.PersistentVolume.Spec.VolumeMode != api.PersistentVolumeFilesystem {
-					t.Errorf("Unexpected volume mode %q", *spec.PersistentVolume.Spec.VolumeMode)
+					t.Errorf("Unexpected volume mode %q", *rec.Spec.PersistentVolume.Spec.VolumeMode)
 				}
 			default:
 				t.Fatal("invalid volume.Spec constructed")

pkg/volume/csi/csi_test.go

@@ -443,14 +443,14 @@ func TestCSI_VolumeAll(t *testing.T) {
 			// ******** Volume Reconstruction ************* //
 			volPath := filepath.Dir(csiMounter.GetPath())
 			t.Log("csiTest.VolumeAll entering plugin.ConstructVolumeSpec for path", volPath)
-			spec, err := volPlug.ConstructVolumeSpec(test.volName, volPath)
+			rec, err := volPlug.ConstructVolumeSpec(test.volName, volPath)
 			if err != nil {
 				t.Fatalf("csiTest.VolumeAll plugin.ConstructVolumeSpec failed: %s", err)
 			} else {
-				if spec == nil {
+				if rec.Spec == nil {
 					t.Fatalf("csiTest.VolumeAll plugin.ConstructVolumeSpec returned nil spec")
 				} else {
-					volSpec = spec
+					volSpec = rec.Spec
 
 					if test.isInline {
 						if volSpec.Volume == nil || volSpec.Volume.CSI == nil {

pkg/volume/downwardapi/downwardapi.go

@@ -123,14 +123,16 @@ func (plugin *downwardAPIPlugin) NewUnmounter(volName string, podUID types.UID)
 	}, nil
 }
 
-func (plugin *downwardAPIPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *downwardAPIPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	downwardAPIVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
 			DownwardAPI: &v1.DownwardAPIVolumeSource{},
 		},
 	}
-	return volume.NewSpecFromVolume(downwardAPIVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(downwardAPIVolume),
+	}, nil
 }
 
 // downwardAPIVolume retrieves downward API data and placing them into the volume on the host.

pkg/volume/emptydir/empty_dir.go

@@ -188,14 +188,16 @@ func (plugin *emptyDirPlugin) newUnmounterInternal(volName string, podUID types.
 	return ed, nil
 }
 
-func (plugin *emptyDirPlugin) ConstructVolumeSpec(volName, mountPath string) (*volume.Spec, error) {
+func (plugin *emptyDirPlugin) ConstructVolumeSpec(volName, mountPath string) (volume.ReconstructedVolume, error) {
 	emptyDirVolume := &v1.Volume{
 		Name: volName,
 		VolumeSource: v1.VolumeSource{
 			EmptyDir: &v1.EmptyDirVolumeSource{},
 		},
 	}
-	return volume.NewSpecFromVolume(emptyDirVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(emptyDirVolume),
+	}, nil
 }
 
 // mountDetector abstracts how to find what kind of mount a path is backed by.

pkg/volume/fc/fc.go

@@ -240,7 +240,7 @@ func (plugin *fcPlugin) newUnmapperInternal(volName string, podUID types.UID, ma
 	}, nil
 }
 
-func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	// Find globalPDPath from pod volume directory(mountPath)
 	// examples:
 	//   mountPath:     pods/{podUid}/volumes/kubernetes.io~fc/{volumeName}
@@ -256,10 +256,10 @@ func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volu
 	if io.IsInconsistentReadError(err) {
 		klog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err)
 		klog.Errorf("Kubelet cannot unmount volume at %s, please unmount it manually", mountPath)
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	for _, path := range paths {
 		if strings.Contains(path, plugin.host.GetPluginDir(fcPluginName)) {
@@ -269,12 +269,12 @@ func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volu
 	}
 	// Couldn't fetch globalPDPath
 	if len(globalPDPath) == 0 {
-		return nil, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
+		return volume.ReconstructedVolume{}, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
 	}
 
 	wwns, lun, wwids, err := parsePDName(globalPDPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve volume plugin information from globalPDPath: %s", err)
+		return volume.ReconstructedVolume{}, fmt.Errorf("failed to retrieve volume plugin information from globalPDPath: %s", err)
 	}
 	// Create volume from wwn+lun or wwid
 	fcVolume := &v1.Volume{
@@ -285,7 +285,9 @@ func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volu
 	}
 	klog.V(5).Infof("ConstructVolumeSpec: TargetWWNs: %v, Lun: %v, WWIDs: %v",
 		fcVolume.VolumeSource.FC.TargetWWNs, *fcVolume.VolumeSource.FC.Lun, fcVolume.VolumeSource.FC.WWIDs)
-	return volume.NewSpecFromVolume(fcVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(fcVolume),
+	}, nil
 }
 
 // ConstructBlockVolumeSpec creates a new volume.Spec with following steps.

pkg/volume/flexvolume/plugin.go

@@ -260,7 +260,7 @@ func (plugin *flexVolumeAttachablePlugin) CanDeviceMount(spec *volume.Spec) (boo
 }
 
 // ConstructVolumeSpec is part of the volume.AttachableVolumePlugin interface.
-func (plugin *flexVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *flexVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	flexVolume := &api.Volume{
 		Name: volumeName,
 		VolumeSource: api.VolumeSource{
@@ -269,7 +269,9 @@ func (plugin *flexVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(flexVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(flexVolume),
+	}, nil
 }
 
 func (plugin *flexVolumePlugin) SupportsMountOption() bool {

pkg/volume/gcepd/gce_pd.go

@@ -299,17 +299,17 @@ func (plugin *gcePersistentDiskPlugin) NodeExpand(resizeOptions volume.NodeResiz
 
 var _ volume.NodeExpandableVolumePlugin = &gcePersistentDiskPlugin{}
 
-func (plugin *gcePersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *gcePersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	mounter := plugin.host.GetMounter(plugin.GetPluginName())
 	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
 	if !ok {
-		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
+		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
 	}
 	hu := kvh.GetHostUtil()
 	pluginMntDir := util.GetPluginMountDir(plugin.host, plugin.GetPluginName())
 	sourceName, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	gceVolume := &v1.Volume{
 		Name: volumeName,
@@ -319,7 +319,9 @@ func (plugin *gcePersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(gceVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(gceVolume),
+	}, nil
 }
 
 // Abstract interface to PD operations.

pkg/volume/git_repo/git_repo.go

@@ -123,14 +123,16 @@ func (plugin *gitRepoPlugin) NewUnmounter(volName string, podUID types.UID) (vol
 	}, nil
 }
 
-func (plugin *gitRepoPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *gitRepoPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	gitVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
 			GitRepo: &v1.GitRepoVolumeSource{},
 		},
 	}
-	return volume.NewSpecFromVolume(gitVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(gitVolume),
+	}, nil
 }
 
 // gitRepo volumes are directories which are pre-filled from a git repository.

pkg/volume/hostpath/host_path.go

@@ -181,7 +181,7 @@ func (plugin *hostPathPlugin) NewProvisioner(options volume.VolumeOptions) (volu
 	return newProvisioner(options, plugin.host, plugin)
 }
 
-func (plugin *hostPathPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *hostPathPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	hostPathVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
@@ -190,7 +190,9 @@ func (plugin *hostPathPlugin) ConstructVolumeSpec(volumeName, mountPath string)
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(hostPathVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(hostPathVolume),
+	}, nil
 }
 
 func newDeleter(spec *volume.Spec, host volume.VolumeHost) (volume.Deleter, error) {

pkg/volume/iscsi/iscsi.go

@@ -221,7 +221,7 @@ func (plugin *iscsiPlugin) newUnmapperInternal(volName string, podUID types.UID,
 	}, nil
 }
 
-func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	// Find globalPDPath from pod volume directory(mountPath)
 	var globalPDPath string
 	mounter := plugin.host.GetMounter(plugin.GetPluginName())
@@ -233,10 +233,10 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v
 	if io.IsInconsistentReadError(err) {
 		klog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err)
 		klog.Errorf("Kubelet cannot unmount volume at %s, please unmount it and all mounts of the same device manually.", mountPath)
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 
 	for _, path := range paths {
@@ -247,25 +247,25 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v
 	}
 	// Couldn't fetch globalPDPath
 	if len(globalPDPath) == 0 {
-		return nil, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
+		return volume.ReconstructedVolume{}, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
 	}
 
 	// Obtain iscsi disk configurations from globalPDPath
 	device, _, err := extractDeviceAndPrefix(globalPDPath)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	bkpPortal, iqn, err := extractPortalAndIqn(device)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	arr := strings.Split(device, "-lun-")
 	if len(arr) < 2 {
-		return nil, fmt.Errorf("failed to retrieve lun from globalPDPath: %v", globalPDPath)
+		return volume.ReconstructedVolume{}, fmt.Errorf("failed to retrieve lun from globalPDPath: %v", globalPDPath)
 	}
 	lun, err := strconv.Atoi(arr[1])
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	iface, _ := extractIface(globalPDPath)
 	iscsiVolume := &v1.Volume{
@@ -279,7 +279,9 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(iscsiVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(iscsiVolume),
+	}, nil
 }
 
 func (plugin *iscsiPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {

pkg/volume/local/local.go

@@ -197,7 +197,7 @@ func (plugin *localVolumePlugin) NewBlockVolumeUnmapper(volName string,
 }
 
 // TODO: check if no path and no topology constraints are ok
-func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	fs := v1.PersistentVolumeFilesystem
 	// The main purpose of reconstructed volume is to clean unused mount points
 	// and directories.
@@ -209,7 +209,7 @@ func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath strin
 	mounter := plugin.host.GetMounter(plugin.GetPluginName())
 	refs, err := mounter.GetMountRefs(mountPath)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	baseMountPath := plugin.generateBlockDeviceBaseGlobalPath()
 	for _, ref := range refs {
@@ -221,7 +221,7 @@ func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath strin
 			// source and can be used in reconstructed volume.
 			path, _, err = mount.GetDeviceNameFromMount(mounter, ref)
 			if err != nil {
-				return nil, err
+				return volume.ReconstructedVolume{}, err
 			}
 			klog.V(4).Infof("local: reconstructing volume %q (pod volume mount: %q) with device %q", volumeName, mountPath, path)
 			break
@@ -240,7 +240,9 @@ func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath strin
 			VolumeMode: &fs,
 		},
 	}
-	return volume.NewSpecFromPersistentVolume(localVolume, false), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromPersistentVolume(localVolume, false),
+	}, nil
 }
 
 func (plugin *localVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName,

pkg/volume/local/local_test.go

@@ -543,34 +543,34 @@ func TestConstructVolumeSpec(t *testing.T) {
 			}
 			mounter.(*mount.FakeMounter).MountPoints = fakeMountPoints
 			volPath := filepath.Join(tmpDir, testMountPath)
-			spec, err := plug.ConstructVolumeSpec(testPVName, volPath)
+			rec, err := plug.ConstructVolumeSpec(testPVName, volPath)
 			if err != nil {
 				t.Errorf("ConstructVolumeSpec() failed: %v", err)
 			}
-			if spec == nil {
+			if rec.Spec == nil {
 				t.Fatalf("ConstructVolumeSpec() returned nil")
 			}
 
-			volName := spec.Name()
+			volName := rec.Spec.Name()
 			if volName != testPVName {
 				t.Errorf("Expected volume name %q, got %q", testPVName, volName)
 			}
 
-			if spec.Volume != nil {
+			if rec.Spec.Volume != nil {
 				t.Errorf("Volume object returned, expected nil")
 			}
 
-			pv := spec.PersistentVolume
+			pv := rec.Spec.PersistentVolume
 			if pv == nil {
 				t.Fatalf("PersistentVolume object nil")
 			}
 
-			if spec.PersistentVolume.Spec.VolumeMode == nil {
+			if rec.Spec.PersistentVolume.Spec.VolumeMode == nil {
 				t.Fatalf("Volume mode has not been set.")
 			}
 
-			if *spec.PersistentVolume.Spec.VolumeMode != v1.PersistentVolumeFilesystem {
+			if *rec.Spec.PersistentVolume.Spec.VolumeMode != v1.PersistentVolumeFilesystem {
-				t.Errorf("Unexpected volume mode %q", *spec.PersistentVolume.Spec.VolumeMode)
+				t.Errorf("Unexpected volume mode %q", *rec.Spec.PersistentVolume.Spec.VolumeMode)
 			}
 
 			ls := pv.Spec.PersistentVolumeSource.Local

pkg/volume/nfs/nfs.go

@@ -176,7 +176,7 @@ func (plugin *nfsPlugin) Recycle(pvName string, spec *volume.Spec, eventRecorder
 	return recyclerclient.RecycleVolumeByWatchingPodUntilCompletion(pvName, pod, plugin.host.GetKubeClient(), eventRecorder)
 }
 
-func (plugin *nfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *nfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	nfsVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
@@ -185,7 +185,9 @@ func (plugin *nfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*vol
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(nfsVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(nfsVolume),
+	}, nil
 }
 
 // NFS volumes represent a bare host file or directory mount of an NFS export.

pkg/volume/noop_expandable_plugin.go

@@ -60,8 +60,8 @@ func (n *noopExpandableVolumePluginInstance) NewUnmounter(name string, podUID ty
 	return nil, nil
 }
 
-func (n *noopExpandableVolumePluginInstance) ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error) {
+func (n *noopExpandableVolumePluginInstance) ConstructVolumeSpec(volumeName, mountPath string) (ReconstructedVolume, error) {
-	return n.spec, nil
+	return ReconstructedVolume{Spec: n.spec}, nil
 }
 
 func (n *noopExpandableVolumePluginInstance) SupportsMountOption() bool {

pkg/volume/plugins.go

@@ -166,7 +166,7 @@ type VolumePlugin interface {
 	// and volumePath. The spec may have incomplete information due to limited
 	// information from input. This function is used by volume manager to reconstruct
 	// volume spec by reading the volume directories from disk
-	ConstructVolumeSpec(volumeName, volumePath string) (*Spec, error)
+	ConstructVolumeSpec(volumeName, volumePath string) (ReconstructedVolume, error)
 
 	// SupportsMountOption returns true if volume plugins supports Mount options
 	// Specifying mount options in a volume plugin that doesn't support
@@ -570,6 +570,12 @@ type VolumeConfig struct {
 	ProvisioningEnabled bool
 }
 
+// ReconstructedVolume contains information about a volume reconstructed by
+// ConstructVolumeSpec().
+type ReconstructedVolume struct {
+	Spec *Spec
+}
+
 // NewSpecFromVolume creates an Spec from an v1.Volume
 func NewSpecFromVolume(vs *v1.Volume) *Spec {
 	return &Spec{

pkg/volume/plugins_test.go

@@ -99,8 +99,8 @@ func (plugin *testPlugins) NewUnmounter(name string, podUID types.UID) (Unmounte
 	return nil, nil
 }
 
-func (plugin *testPlugins) ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error) {
+func (plugin *testPlugins) ConstructVolumeSpec(volumeName, mountPath string) (ReconstructedVolume, error) {
-	return nil, nil
+	return ReconstructedVolume{}, nil
 }
 
 func newTestPlugin() []VolumePlugin {

pkg/volume/portworx/portworx.go

@@ -210,7 +210,7 @@ func (plugin *portworxVolumePlugin) ExpandVolumeDevice(
 	return newSize, nil
 }
 
-func (plugin *portworxVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *portworxVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	portworxVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
@@ -219,7 +219,9 @@ func (plugin *portworxVolumePlugin) ConstructVolumeSpec(volumeName, mountPath st
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(portworxVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(portworxVolume),
+	}, nil
 }
 
 func (plugin *portworxVolumePlugin) SupportsMountOption() bool {

pkg/volume/projected/projected.go

@@ -135,7 +135,7 @@ func (plugin *projectedPlugin) NewUnmounter(volName string, podUID types.UID) (v
 	}, nil
 }
 
-func (plugin *projectedPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *projectedPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	projectedVolume := &v1.Volume{
 		Name: volumeName,
 		VolumeSource: v1.VolumeSource{
@@ -143,7 +143,9 @@ func (plugin *projectedPlugin) ConstructVolumeSpec(volumeName, mountPath string)
 		},
 	}
 
-	return volume.NewSpecFromVolume(projectedVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(projectedVolume),
+	}, nil
 }
 
 type projectedVolume struct {

pkg/volume/rbd/rbd.go

@@ -386,17 +386,17 @@ func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID,
 	}, nil
 }
 
-func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	mounter := plugin.host.GetMounter(plugin.GetPluginName())
 	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
 	if !ok {
-		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
+		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
 	}
 	hu := kvh.GetHostUtil()
 	pluginMntDir := volutil.GetPluginMountDir(plugin.host, plugin.GetPluginName())
 	sourceName, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	s := dstrings.Split(sourceName, "-image-")
 	if len(s) != 2 {
@@ -414,11 +414,11 @@ func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*vol
 		klog.V(3).Infof("SourceName %s wrong, fallback to old format", sourceName)
 		sourceName, err = plugin.getDeviceNameFromOldMountPath(mounter, mountPath)
 		if err != nil {
-			return nil, err
+			return volume.ReconstructedVolume{}, err
 		}
 		s = dstrings.Split(sourceName, "-image-")
 		if len(s) != 2 {
-			return nil, fmt.Errorf("sourceName %s wrong, should be pool+\"-image-\"+imageName", sourceName)
+			return volume.ReconstructedVolume{}, fmt.Errorf("sourceName %s wrong, should be pool+\"-image-\"+imageName", sourceName)
 		}
 	}
 	rbdVolume := &v1.Volume{
@@ -430,7 +430,9 @@ func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*vol
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(rbdVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(rbdVolume),
+	}, nil
 }
 
 func (plugin *rbdPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {

pkg/volume/rbd/rbd_test.go

@@ -631,15 +631,15 @@ func TestConstructVolumeSpec(t *testing.T) {
 		if err = fakeMounter.Mount(c.targetPath, podMountPath, "fake", []string{"bind"}); err != nil {
 			t.Fatalf("Mount %s to %s failed: %v", c.targetPath, podMountPath, err)
 		}
-		spec, err := plug.ConstructVolumeSpec(c.volumeName, podMountPath)
+		rec, err := plug.ConstructVolumeSpec(c.volumeName, podMountPath)
 		if err != nil {
 			t.Errorf("ConstructVolumeSpec failed: %v", err)
 		} else {
-			if spec.Volume.RBD.RBDPool != pool {
+			if rec.Spec.Volume.RBD.RBDPool != pool {
-				t.Errorf("Mismatch rbd pool: wanted %s, got %s", pool, spec.Volume.RBD.RBDPool)
+				t.Errorf("Mismatch rbd pool: wanted %s, got %s", pool, rec.Spec.Volume.RBD.RBDPool)
 			}
-			if spec.Volume.RBD.RBDImage != image {
+			if rec.Spec.Volume.RBD.RBDImage != image {
-				t.Fatalf("Mismatch rbd image: wanted %s, got %s", image, spec.Volume.RBD.RBDImage)
+				t.Fatalf("Mismatch rbd image: wanted %s, got %s", image, rec.Spec.Volume.RBD.RBDImage)
 			}
 		}
 		if err = fakeMounter.Unmount(podMountPath); err != nil {

pkg/volume/secret/secret.go

@@ -125,7 +125,7 @@ func (plugin *secretPlugin) NewUnmounter(volName string, podUID types.UID) (volu
 	}, nil
 }
 
-func (plugin *secretPlugin) ConstructVolumeSpec(volName, mountPath string) (*volume.Spec, error) {
+func (plugin *secretPlugin) ConstructVolumeSpec(volName, mountPath string) (volume.ReconstructedVolume, error) {
 	secretVolume := &v1.Volume{
 		Name: volName,
 		VolumeSource: v1.VolumeSource{
@@ -134,7 +134,9 @@ func (plugin *secretPlugin) ConstructVolumeSpec(volName, mountPath string) (*vol
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(secretVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(secretVolume),
+	}, nil
 }
 
 type secretVolume struct {

pkg/volume/testing/testing.go

@@ -451,10 +451,12 @@ func (plugin *FakeVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode
 	return []v1.PersistentVolumeAccessMode{}
 }
 
-func (plugin *FakeVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *FakeVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
-	return &volume.Spec{
+	return volume.ReconstructedVolume{
-		Volume: &v1.Volume{
+		Spec: &volume.Spec{
-			Name: volumeName,
+			Volume: &v1.Volume{
+				Name: volumeName,
+			},
 		},
 	}, nil
 }
@@ -526,7 +528,7 @@ func (f *FakeBasicVolumePlugin) CanSupport(spec *volume.Spec) bool {
 	return strings.HasPrefix(spec.Name(), f.GetPluginName())
 }
 
-func (f *FakeBasicVolumePlugin) ConstructVolumeSpec(ame, mountPath string) (*volume.Spec, error) {
+func (f *FakeBasicVolumePlugin) ConstructVolumeSpec(ame, mountPath string) (volume.ReconstructedVolume, error) {
 	return f.Plugin.ConstructVolumeSpec(ame, mountPath)
 }
 
@@ -647,8 +649,8 @@ func (plugin *FakeFileVolumePlugin) NewUnmounter(name string, podUID types.UID)
 	return nil, nil
 }
 
-func (plugin *FakeFileVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *FakeFileVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
-	return nil, nil
+	return volume.ReconstructedVolume{}, nil
 }
 
 func NewFakeFileVolumePlugin() []volume.VolumePlugin {

pkg/volume/util/operationexecutor/operation_executor.go

@@ -150,7 +150,7 @@ type OperationExecutor interface {
 	// ExpandInUseVolume will resize volume's file system to expected size without unmounting the volume.
 	ExpandInUseVolume(volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, currentSize resource.Quantity) error
 	// ReconstructVolumeOperation construct a new volumeSpec and returns it created by plugin
-	ReconstructVolumeOperation(volumeMode v1.PersistentVolumeMode, plugin volume.VolumePlugin, mapperPlugin volume.BlockVolumePlugin, uid types.UID, podName volumetypes.UniquePodName, volumeSpecName string, volumePath string, pluginName string) (*volume.Spec, error)
+	ReconstructVolumeOperation(volumeMode v1.PersistentVolumeMode, plugin volume.VolumePlugin, mapperPlugin volume.BlockVolumePlugin, uid types.UID, podName volumetypes.UniquePodName, volumeSpecName string, volumePath string, pluginName string) (volume.ReconstructedVolume, error)
 	// CheckVolumeExistenceOperation checks volume existence
 	CheckVolumeExistenceOperation(volumeSpec *volume.Spec, mountPath, volumeName string, mounter mount.Interface, uniqueVolumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, podUID types.UID, attachable volume.AttachableVolumePlugin) (bool, error)
 }
@@ -1061,17 +1061,17 @@ func (oe *operationExecutor) ReconstructVolumeOperation(
 	podName volumetypes.UniquePodName,
 	volumeSpecName string,
 	volumePath string,
-	pluginName string) (*volume.Spec, error) {
+	pluginName string) (volume.ReconstructedVolume, error) {
 
 	// Filesystem Volume case
 	if volumeMode == v1.PersistentVolumeFilesystem {
 		// Create volumeSpec from mount path
 		klog.V(5).Infof("Starting operationExecutor.ReconstructVolume for file volume on pod %q", podName)
-		volumeSpec, err := plugin.ConstructVolumeSpec(volumeSpecName, volumePath)
+		reconstructed, err := plugin.ConstructVolumeSpec(volumeSpecName, volumePath)
 		if err != nil {
-			return nil, err
+			return volume.ReconstructedVolume{}, err
 		}
-		return volumeSpec, nil
+		return reconstructed, nil
 	}
 
 	// Block Volume case
@@ -1083,9 +1083,11 @@ func (oe *operationExecutor) ReconstructVolumeOperation(
 	// ex. volumePath: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
 	volumeSpec, err := mapperPlugin.ConstructBlockVolumeSpec(uid, volumeSpecName, volumePath)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
-	return volumeSpec, nil
+	return volume.ReconstructedVolume{
+		Spec: volumeSpec,
+	}, nil
 }
 
 // CheckVolumeExistenceOperation checks mount path directory if volume still exists

pkg/volume/vsphere_volume/vsphere_volume.go

@@ -153,17 +153,17 @@ func (plugin *vsphereVolumePlugin) newUnmounterInternal(volName string, podUID t
 		}}, nil
 }
 
-func (plugin *vsphereVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+func (plugin *vsphereVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
 	mounter := plugin.host.GetMounter(plugin.GetPluginName())
 	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
 	if !ok {
-		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
+		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
 	}
 	hu := kvh.GetHostUtil()
 	pluginMntDir := util.GetPluginMountDir(plugin.host, plugin.GetPluginName())
 	volumePath, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
 	if err != nil {
-		return nil, err
+		return volume.ReconstructedVolume{}, err
 	}
 	volumePath = strings.Replace(volumePath, "\\040", " ", -1)
 	klog.V(5).Infof("vSphere volume path is %q", volumePath)
@@ -175,7 +175,9 @@ func (plugin *vsphereVolumePlugin) ConstructVolumeSpec(volumeName, mountPath str
 			},
 		},
 	}
-	return volume.NewSpecFromVolume(vsphereVolume), nil
+	return volume.ReconstructedVolume{
+		Spec: volume.NewSpecFromVolume(vsphereVolume),
+	}, nil
 }
 
 // Abstract interface to disk operations.