Fix volume states out of sync problem after kubelet restarts
When kubelet restarts, all the information about the volumes will be gone from actual/desired states. When update node status with mounted volumes, the volume list might be empty although there are still volumes are mounted and in turn causing master to detach those volumes since they are not in the mounted volumes list. This fix is to make sure only update mounted volumes list after reconciler starts sync states process. This sync state process will scan the existing volume directories and reconstruct actual states if they are missing. This PR also fixes the problem during orphaned pods' directories. In case of the pod directory is unmounted but has not yet deleted (e.g., interrupted with kubelet restarts), clean up routine will delete the directory so that the pod directoriy could be cleaned up (it is safe to delete directory since it is no longer mounted) The third issue this PR fixes is that during reconstruct volume in actual state, mounter could not be nil since it is required for creating container.VolumeMap. If it is nil, it might cause nil pointer exception in kubelet. Details are in proposal PR #33203
This commit is contained in:
@@ -239,9 +239,9 @@ func (kl *Kubelet) GetExtraSupplementalGroupsForPod(pod *api.Pod) []int64 {
|
||||
return kl.volumeManager.GetExtraSupplementalGroupsForPod(pod)
|
||||
}
|
||||
|
||||
// getPodVolumeNameListFromDisk returns a list of the volume names by reading the
|
||||
// getPodVolumePathListFromDisk returns a list of the volume paths by reading the
|
||||
// volume directories for the given pod from the disk.
|
||||
func (kl *Kubelet) getPodVolumeNameListFromDisk(podUID types.UID) ([]string, error) {
|
||||
func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, error) {
|
||||
volumes := []string{}
|
||||
podVolDir := kl.getPodVolumesDir(podUID)
|
||||
volumePluginDirs, err := ioutil.ReadDir(podVolDir)
|
||||
@@ -254,9 +254,11 @@ func (kl *Kubelet) getPodVolumeNameListFromDisk(podUID types.UID) ([]string, err
|
||||
volumePluginPath := path.Join(podVolDir, volumePluginName)
|
||||
volumeDirs, err := util.ReadDirNoStat(volumePluginPath)
|
||||
if err != nil {
|
||||
return volumes, err
|
||||
return volumes, fmt.Errorf("Could not read directory %s: %v", volumePluginPath, err)
|
||||
}
|
||||
for _, volumeDir := range volumeDirs {
|
||||
volumes = append(volumes, path.Join(volumePluginPath, volumeDir))
|
||||
}
|
||||
volumes = append(volumes, volumeDirs...)
|
||||
}
|
||||
return volumes, nil
|
||||
}
|
||||
|
Reference in New Issue
Block a user