github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

force rbd image unlock if the image is not used

Browse Source 
Signed-off-by: Huamin Chen <hchen@redhat.com>
This commit is contained in:
Huamin Chen
2017-02-16 15:55:55 -05:00
parent 2509ab0c7a
commit 71406aa4a6
1 changed files with 54 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
pkg/volume/rbd/rbd_util.go
View File

@@ -43,6 +43,7 @@ import (
const (
imageWatcherStr = "watcher="
kubeLockMagic = "kubelet_lock_magic_"
)
// search /sys/bus for rbd device that matches given pool and image
@@ -114,8 +115,15 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
} else {
secret_opt = []string{"-k", b.Keyring}
}
if len(b.adminId) == 0 {
b.adminId = b.Id
}
if len(b.adminSecret) == 0 {
b.adminSecret = b.Secret
}
// construct lock id using host name and a magic prefix
lock_id := "kubelet_lock_magic_" + node.GetHostname("")
lock_id := kubeLockMagic + node.GetHostname("")
l := len(b.Mon)
// avoid mount storm, pick a host randomly
@@ -130,7 +138,7 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
cmd, err = b.plugin.execCommand("rbd",
append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
output = string(cmd)
glog.Infof("lock list output %q", output)
if err != nil {
continue
}
@@ -142,6 +150,23 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
glog.V(1).Infof("rbd: lock already held for %s", lock_id)
return nil
}
// clean up orphaned lock if no watcher on the image
used, statusErr := util.rbdStatus(&b)
if statusErr == nil && !used {
re := regexp.MustCompile("client.* " + kubeLockMagic + ".*")
locks := re.FindAllStringSubmatch(output, -1)
for _, v := range locks {
if len(v) > 0 {
lockInfo := strings.Split(v[0], " ")
if len(lockInfo) > 2 {
cmd, err = b.plugin.execCommand("rbd",
append([]string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd))
}
}
}
}
// hold a lock: rbd lock add
cmd, err = b.plugin.execCommand("rbd",
append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
@@ -220,6 +245,20 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
var err error
var output []byte
// create mount point
globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
}
if !notMnt {
return nil
}
if err = os.MkdirAll(globalPDPath, 0750); err != nil {
return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
}
devicePath, found := waitForPath(b.Pool, b.Image, 1)
if !found {
// modprobe
@@ -227,6 +266,18 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
if err != nil {
return fmt.Errorf("rbd: failed to modprobe rbd error:%v", err)
}
// fence off other mappers
if err = util.fencing(b); err != nil {
return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
}
// rbd lock remove needs ceph and image config
// but kubelet doesn't get them from apiserver during teardown
// so persit rbd config so upon disk detach, rbd lock can be removed
// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
// the json file remains invisible during rbd mount and thus won't be removed accidentally.
util.persistRBD(b, globalPDPath)
// rbd map
l := len(b.Mon)
// avoid mount storm, pick a host randomly
@@ -255,34 +306,8 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error {
return errors.New("Could not map image: Timeout after 10s")
}
}
// mount it
globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
}
if !notMnt {
return nil
}
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
}
// fence off other mappers
if err := util.fencing(b); err != nil {
// rbd unmap before exit
b.plugin.execCommand("rbd", []string{"unmap", devicePath})
return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
}
// rbd lock remove needs ceph and image config
// but kubelet doesn't get them from apiserver during teardown
// so persit rbd config so upon disk detach, rbd lock can be removed
// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
// the json file remains invisible during rbd mount and thus won't be removed accidentally.
util.persistRBD(b, globalPDPath)
if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil {
err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
}