github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Retry reading /proc/mounts indifinetly in FC and iSCSI volume reconstruction

Browse Source 
iSCSI and FC volume plugins do not implement real 3rd party attach/detach.
If reconstruction fails with an error on a FC or iSCSI volume, it will not
be unmounted from the volume global dir and at the same time it will be
marked as unused, to be available to be mounted on another node.

The volume can then be mounted on several nodes, resulting in volume
corruption.

The other block based volume plugins implement attach/detach that either
makes the volume stuck (can't be detached) or will be force-detached from a
node before attaching it somewhere else.
This commit is contained in:
Jan Safranek
2021-05-19 18:11:52 +02:00
parent 82cfe9f14f
commit a95842095e
3 changed files with 48 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
pkg/volume/util/util.go
View File

@@ -20,18 +20,12 @@ import (
"context"
"fmt"
"io/ioutil"
storagehelpers "k8s.io/component-helpers/storage/volume"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"k8s.io/component-helpers/scheduling/corev1"
"k8s.io/klog/v2"
"k8s.io/mount-utils"
utilexec "k8s.io/utils/exec"
utilstrings "k8s.io/utils/strings"
"time"
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
@@ -40,13 +34,21 @@ import (
apiruntime "k8s.io/apimachinery/pkg/runtime"
utypes "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/component-helpers/scheduling/corev1"
storagehelpers "k8s.io/component-helpers/storage/volume"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/api/legacyscheme"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/securitycontext"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
"k8s.io/mount-utils"
utilexec "k8s.io/utils/exec"
"k8s.io/utils/io"
utilstrings "k8s.io/utils/strings"
)
const (
@@ -732,3 +734,29 @@ func IsDeviceMountableVolume(volumeSpec *volume.Spec, volumePluginMgr *volume.Vo
return false
}
// GetReliableMountRefs calls mounter.GetMountRefs and retries on IsInconsistentReadError.
// To be used in volume reconstruction of volume plugins that don't have any protection
// against mounting a single volume on multiple nodes (such as attach/detach).
func GetReliableMountRefs(mounter mount.Interface, mountPath string) ([]string, error) {
var paths []string
var lastErr error
err := wait.PollImmediate(10*time.Millisecond, time.Minute, func() (bool, error) {
var err error
paths, err = mounter.GetMountRefs(mountPath)
if io.IsInconsistentReadError(err) {
lastErr = err
return false, nil
}
if err != nil {
return false, err
}
return true, nil
})
if err == wait.ErrWaitTimeout {
klog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err)
klog.Errorf("Kubelet cannot unmount volume at %s, please unmount it manually", mountPath)
return nil, lastErr
}
return paths, err
}