Add e2e test to check for filesystem volume device mount cleanup

pkg/util/mount/mount.go

@@ -48,7 +48,7 @@ type Interface interface {
 	// most notably linux bind mounts and symbolic link.
 	IsLikelyNotMountPoint(file string) (bool, error)
 	// GetMountRefs finds all mount references to the path, returns a
-	// list of paths. Path could be a mountpoint or a normal
+	// list of paths. Path could be a mountpoint path, device or a normal
 	// directory (for bind mount).
 	GetMountRefs(pathname string) ([]string, error)
 }

pkg/util/mount/mount_linux.go

@@ -238,7 +238,7 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
 }
 
 // GetMountRefs finds all mount references to pathname, returns a
-// list of paths. Path could be a mountpoint or a normal
+// list of paths. Path could be a mountpoint path, device or a normal
 // directory (for bind mount).
 func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
 	pathExists, pathErr := PathExists(pathname)
@@ -441,8 +441,7 @@ func parseProcMounts(content []byte) ([]MountPoint, error) {
 
 // SearchMountPoints finds all mount references to the source, returns a list of
 // mountpoints.
-// The source can be a mount point or a normal directory (bind mount). We
+// This function assumes source cannot be device.
-// didn't support device because there is no use case by now.
 // Some filesystems may share a source name, e.g. tmpfs. And for bind mounting,
 // it's possible to mount a non-root path of a filesystem, so we need to use
 // root path and major:minor to represent mount source uniquely.

test/e2e/storage/testsuites/subpath.go

@@ -157,24 +157,24 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 	}
 
 	cleanup := func() {
-		if l.pod != nil {
+		// if l.pod != nil {
-			ginkgo.By("Deleting pod")
+		// ginkgo.By("Deleting pod")
-			err := e2epod.DeletePodWithWait(f.ClientSet, l.pod)
+		// err := e2epod.DeletePodWithWait(f.ClientSet, l.pod)
-			framework.ExpectNoError(err, "while deleting pod")
+		// framework.ExpectNoError(err, "while deleting pod")
-			l.pod = nil
+		// l.pod = nil
-		}
+		// }
 
-		if l.resource != nil {
+		// if l.resource != nil {
-			l.resource.cleanupResource()
+		// l.resource.cleanupResource()
-			l.resource = nil
+		// l.resource = nil
-		}
+		// }
 
-		if l.driverCleanup != nil {
+		// if l.driverCleanup != nil {
-			l.driverCleanup()
+		// l.driverCleanup()
-			l.driverCleanup = nil
+		// l.driverCleanup = nil
-		}
+		// }
 
-		validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps)
+		// validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps)
 	}
 
 	ginkgo.It("should support non-existent path", func() {
@@ -902,7 +902,7 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete
 	pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
 	framework.ExpectNoError(err, "while getting pod")
 
-	utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true)
+	utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true, true)
 }
 
 func formatVolume(f *framework.Framework, pod *v1.Pod) {

test/e2e/storage/utils/BUILD

@@ -1,9 +1,6 @@
 package(default_visibility = ["//visibility:public"])
 
-load(
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-)
 
 go_library(
     name = "go_default_library",
@@ -16,6 +13,7 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/test/e2e/storage/utils",
     deps = [
+        "//pkg/util/mount:go_default_library",
         "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
@@ -23,6 +21,7 @@ go_library(
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
@@ -49,3 +48,10 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["utils_test.go"],
+    embed = [":go_default_library"],
+    deps = ["//vendor/github.com/onsi/gomega:go_default_library"],
+)

test/e2e/storage/utils/utils.go

@@ -20,7 +20,9 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -34,6 +36,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -234,9 +237,48 @@ func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Frame
 	framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path)
 }
 
+// findGlobalVolumeMountPaths finds all global volume mount paths for given pod from the host mount information.
+// This function assumes:
+// 1) pod volume mount paths exists in /var/lib/kubelet/pods/<pod-uid>/volumes/
+// 2) global volume mount paths exists in /var/lib/kubelet/plugins/
+func findGlobalVolumeMountPaths(mountInfo string, podUID string) ([]string, error) {
+	tmpfile, err := ioutil.TempFile("", "mountinfo")
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+	err = ioutil.WriteFile(tmpfile.Name(), []byte(mountInfo), 0644)
+	if err != nil {
+		return nil, err
+	}
+	podVolumeMountBase := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/", podUID)
+	globalVolumeMountBase := "/var/lib/kubelet/plugins"
+	mis, err := mount.ParseMountInfo(tmpfile.Name())
+	if err != nil {
+		return nil, err
+	}
+	globalVolumeMountPaths := []string{}
+	for _, mi := range mis {
+		if mount.PathWithinBase(mi.MountPoint, podVolumeMountBase) {
+			refs, err := mount.SearchMountPoints(mi.MountPoint, tmpfile.Name())
+			if err != nil {
+				return nil, err
+			}
+			for _, ref := range refs {
+				if mount.PathWithinBase(ref, globalVolumeMountBase) {
+					globalVolumeMountPaths = append(globalVolumeMountPaths, ref)
+				}
+			}
+		}
+	}
+	return globalVolumeMountPaths, nil
+}
+
 // TestVolumeUnmountsFromDeletedPodWithForceOption tests that a volume unmounts if the client pod was deleted while the kubelet was down.
 // forceDelete is true indicating whether the pod is forcefully deleted.
-func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool) {
+// checkSubpath is true indicating whether the subpath should be checked.
+// checkGlobalMount is true indicating whether the global mount should be checked.
+func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool, checkGlobalMount bool) {
 	nodeIP, err := framework.GetHostAddress(c, clientPod)
 	framework.ExpectNoError(err)
 	nodeIP = nodeIP + ":22"
@@ -255,6 +297,24 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
 		gomega.Expect(result.Code).To(gomega.BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
 	}
 
+	var globalVolumeMountPaths []string
+	if checkGlobalMount {
+		// Find global mount path and verify it will be unmounted later.
+		// We don't verify it must exist because:
+		// 1) not all volume types have global mount path, e.g. local filesystem volume with directory source
+		// 2) volume types which failed to mount global mount path will fail in other test
+		ginkgo.By("Find the volume global mount paths")
+		result, err = e2essh.SSH("cat /proc/self/mountinfo", nodeIP, framework.TestContext.Provider)
+		framework.ExpectNoError(err, "Encountered SSH error.")
+		globalVolumeMountPaths, err = findGlobalVolumeMountPaths(result.Stdout, string(clientPod.UID))
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to get global volume mount paths: %v", err))
+		if len(globalVolumeMountPaths) > 0 {
+			framework.Logf("Volume global mount paths found at %v", globalVolumeMountPaths)
+		} else {
+			framework.Logf("No volume global mount paths found")
+		}
+	}
+
 	// This command is to make sure kubelet is started after test finishes no matter it fails or not.
 	defer func() {
 		KubeletCommand(KStart, c, clientPod)
@@ -298,16 +358,28 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
 		gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty (i.e. no subpath mount found).")
 		framework.Logf("Subpath volume unmounted on node %s", clientPod.Spec.NodeName)
 	}
+
+	if checkGlobalMount && len(globalVolumeMountPaths) > 0 {
+		globalMountPathCmd := fmt.Sprintf("ls %s | grep '.'", strings.Join(globalVolumeMountPaths, " "))
+		if isSudoPresent(nodeIP, framework.TestContext.Provider) {
+			globalMountPathCmd = fmt.Sprintf("sudo sh -c \"%s\"", globalMountPathCmd)
+		}
+		ginkgo.By("Expecting the volume global mount path not to be found.")
+		result, err = e2essh.SSH(globalMountPathCmd, nodeIP, framework.TestContext.Provider)
+		e2essh.LogResult(result)
+		framework.ExpectNoError(err, "Encountered SSH error.")
+		gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty.")
+	}
 }
 
 // TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
 func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
-	TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false)
+	TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false, false)
 }
 
 // TestVolumeUnmountsFromForceDeletedPod tests that a volume unmounts if the client pod was forcefully deleted while the kubelet was down.
 func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
-	TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false)
+	TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false, false)
 }
 
 // TestVolumeUnmapsFromDeletedPodWithForceOption tests that a volume unmaps if the client pod was deleted while the kubelet was down.

test/e2e/storage/utils/utils_test.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"testing"
+
+	"github.com/onsi/gomega"
+)
+
+func TestFindGlobalVolumeMountPaths(t *testing.T) {
+	tests := []struct {
+		name      string
+		mountInfo string
+		podUID    string
+		expected  []string
+	}{
+		{
+			name: "pod uses local filesystem pv with block source",
+			mountInfo: `1045 245 0:385 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~secret/default-token-djlt2 rw,relatime shared:199 - tmpfs tmpfs rw
+1047 245 7:6 / /var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered
+1048 245 7:6 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~local-volume/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered
+1054 245 7:6 /provisioning-9823 /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volume-subpaths/local-wdx8b/test-container-subpath-local-preprovisionedpv-d72p/0 rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered
+`,
+			podUID: "ff5e9fa2-7111-486d-854c-848bcc6b3819",
+			expected: []string{
+				"/var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b",
+			},
+		},
+	}
+
+	g := gomega.NewWithT(t)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mountPaths, err := findGlobalVolumeMountPaths(tt.mountInfo, tt.podUID)
+			if err != nil {
+				t.Fatal(err)
+			}
+			g.Expect(mountPaths).To(gomega.ConsistOf(tt.expected))
+		})
+	}
+}