diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 8b90ed13e30..c19b88a80c1 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -108,6 +108,7 @@ go_library( "//pkg/volume/scaleio:go_default_library", "//pkg/volume/secret:go_default_library", "//pkg/volume/storageos:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//pkg/volume/vsphere_volume:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 5b80efe5bfc..b7004673d0e 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -95,6 +95,7 @@ import ( "k8s.io/kubernetes/pkg/util/rlimit" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/version/verflag" + "k8s.io/kubernetes/pkg/volume/util/subpath" nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned" "k8s.io/utils/exec" "k8s.io/utils/nsenter" @@ -364,6 +365,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err } mounter := mount.New(s.ExperimentalMounterPath) + subpather := subpath.New(mounter) var pluginRunner = exec.New() if s.Containerized { klog.V(2).Info("Running kubelet in containerized mode") @@ -372,6 +374,8 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err return nil, err } mounter = mount.NewNsenterMounter(s.RootDirectory, ne) + // NSenter only valid on Linux + subpather = subpath.NewNSEnter(mounter, ne, s.RootDirectory) // an exec interface which can use nsenter for flex plugin calls pluginRunner, err = nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New()) if err != nil { @@ -399,6 +403,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err CSIClient: nil, EventClient: nil, Mounter: mounter, + Subpather: subpather, OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, VolumePlugins: ProbeVolumePlugins(), diff --git a/pkg/controller/.import-restrictions b/pkg/controller/.import-restrictions index e3b8394d058..67877dea147 100644 --- a/pkg/controller/.import-restrictions +++ b/pkg/controller/.import-restrictions @@ -167,6 +167,7 @@ "AllowedPrefixes": [ "k8s.io/kubernetes/pkg/api/legacyscheme", "k8s.io/kubernetes/pkg/api/v1/endpoints", + "k8s.io/kubernetes/pkg/api/v1/node", "k8s.io/kubernetes/pkg/api/v1/pod", "k8s.io/kubernetes/pkg/apis/apps/v1", "k8s.io/kubernetes/pkg/apis/autoscaling", @@ -241,6 +242,7 @@ "k8s.io/kubernetes/pkg/volume/util", "k8s.io/kubernetes/pkg/volume/util/operationexecutor", "k8s.io/kubernetes/pkg/volume/util/recyclerclient", + "k8s.io/kubernetes/pkg/volume/util/subpath", "k8s.io/kubernetes/pkg/volume/util/types", "k8s.io/kubernetes/pkg/volume/util/volumepathhandler", "k8s.io/kubernetes/pkg/api/service", diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index f68704cfff1..3128c2b3424 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -22,6 +22,7 @@ go_library( "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//pkg/volume/util/volumepathhandler:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index d2dd7b3de62..66a4c6c66c3 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -52,6 +52,7 @@ import ( "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" + "k8s.io/kubernetes/pkg/volume/util/subpath" "k8s.io/kubernetes/pkg/volume/util/volumepathhandler" ) @@ -768,3 +769,8 @@ func (adc *attachDetachController) GetEventRecorder() record.EventRecorder { func (adc *attachDetachController) GetCSIClient() csiclient.Interface { return adc.csiClient } + +func (adc *attachDetachController) GetSubpather() subpath.Interface { + // Subpaths not needed in attachdetach controller + return nil +} diff --git a/pkg/controller/volume/expand/BUILD b/pkg/controller/volume/expand/BUILD index c633196b325..63bca838cd2 100644 --- a/pkg/controller/volume/expand/BUILD +++ b/pkg/controller/volume/expand/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//pkg/volume/util/volumepathhandler:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/controller/volume/expand/expand_controller.go b/pkg/controller/volume/expand/expand_controller.go index d53b953524d..9eb82138a28 100644 --- a/pkg/controller/volume/expand/expand_controller.go +++ b/pkg/controller/volume/expand/expand_controller.go @@ -46,6 +46,7 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" + "k8s.io/kubernetes/pkg/volume/util/subpath" "k8s.io/kubernetes/pkg/volume/util/volumepathhandler" ) @@ -339,3 +340,8 @@ func (expc *expandController) GetCSIClient() csiclientset.Interface { // No volume plugin in expand controller needs csi.storage.k8s.io return nil } + +func (expc *expandController) GetSubpather() subpath.Interface { + // not needed for expand controller + return nil +} diff --git a/pkg/controller/volume/persistentvolume/BUILD b/pkg/controller/volume/persistentvolume/BUILD index 6f46984a647..15e79a48eaf 100644 --- a/pkg/controller/volume/persistentvolume/BUILD +++ b/pkg/controller/volume/persistentvolume/BUILD @@ -33,6 +33,7 @@ go_library( "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/recyclerclient:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", diff --git a/pkg/controller/volume/persistentvolume/volume_host.go b/pkg/controller/volume/persistentvolume/volume_host.go index 6e4fc3fdd38..1baebc9de29 100644 --- a/pkg/controller/volume/persistentvolume/volume_host.go +++ b/pkg/controller/volume/persistentvolume/volume_host.go @@ -30,6 +30,7 @@ import ( "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" vol "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util/subpath" ) // VolumeHost interface implementation for PersistentVolumeController. @@ -136,3 +137,8 @@ func (ctrl *PersistentVolumeController) GetCSIClient() csiclientset.Interface { // No volume plugin needs csi.storage.k8s.io client in PV controller. return nil } + +func (ctrl *PersistentVolumeController) GetSubpather() subpath.Interface { + // No volume plugin needs Subpaths in PV controller. + return nil +} diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 80b7ec96a1c..b824a8dbbb5 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -112,6 +112,7 @@ go_library( "//pkg/volume:go_default_library", "//pkg/volume/csi:go_default_library", "//pkg/volume/util:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//pkg/volume/util/types:go_default_library", "//pkg/volume/util/volumepathhandler:go_default_library", "//pkg/volume/validation:go_default_library", @@ -220,6 +221,7 @@ go_test( "//pkg/volume/host_path:go_default_library", "//pkg/volume/testing:go_default_library", "//pkg/volume/util:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index b413304c377..16d98618682 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -113,6 +113,7 @@ import ( "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi" + "k8s.io/kubernetes/pkg/volume/util/subpath" nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned" utilexec "k8s.io/utils/exec" "k8s.io/utils/integer" @@ -255,6 +256,7 @@ type Dependencies struct { OSInterface kubecontainer.OSInterface PodConfig *config.PodConfig Recorder record.EventRecorder + Subpather subpath.Interface VolumePlugins []volume.VolumePlugin DynamicPluginProber volume.DynamicPluginProber TLSOptions *server.TLSOptions @@ -519,6 +521,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, cgroupsPerQOS: kubeCfg.CgroupsPerQOS, cgroupRoot: kubeCfg.CgroupRoot, mounter: kubeDeps.Mounter, + subpather: kubeDeps.Subpather, maxPods: int(kubeCfg.MaxPods), podsPerCore: int(kubeCfg.PodsPerCore), syncLoopMonitor: atomic.Value{}, @@ -1099,6 +1102,9 @@ type Kubelet struct { // Mounter to use for volumes. mounter mount.Interface + // subpather to execute subpath actions + subpather subpath.Interface + // Manager of non-Runtime containers. containerManager cm.ContainerManager diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index f66b0fef21d..fbdcfff2dc0 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -61,6 +61,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/format" mountutil "k8s.io/kubernetes/pkg/util/mount" volumeutil "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util/subpath" "k8s.io/kubernetes/pkg/volume/util/volumepathhandler" volumevalidation "k8s.io/kubernetes/pkg/volume/validation" "k8s.io/kubernetes/third_party/forked/golang/expansion" @@ -127,7 +128,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol } // makeMounts determines the mount points for the given container. -func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) { +func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, subpather subpath.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) { // Kubernetes only mounts on /etc/hosts if: // - container is not an infrastructure (pause) container // - container is not already mounting on /etc/hosts @@ -206,13 +207,13 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h if err != nil { return nil, cleanupAction, err } - if err := mounter.SafeMakeDir(subPath, volumePath, perm); err != nil { + if err := subpather.SafeMakeDir(subPath, volumePath, perm); err != nil { // Don't pass detailed error back to the user because it could give information about host filesystem klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err) return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name) } } - hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{ + hostPath, cleanupAction, err = subpather.PrepareSafeSubpath(subpath.Subpath{ VolumeMountIndex: i, Path: hostPath, VolumeName: vol.InnerVolumeSpecName, @@ -464,7 +465,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai } opts.Envs = append(opts.Envs, envs...) - mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, opts.Envs) + mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, kl.subpather, opts.Envs) if err != nil { return nil, cleanupAction, err } diff --git a/pkg/kubelet/kubelet_pods_linux_test.go b/pkg/kubelet/kubelet_pods_linux_test.go index f854c5093a7..4a8332f2374 100644 --- a/pkg/kubelet/kubelet_pods_linux_test.go +++ b/pkg/kubelet/kubelet_pods_linux_test.go @@ -29,6 +29,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/util/mount" volumetest "k8s.io/kubernetes/pkg/volume/testing" + "k8s.io/kubernetes/pkg/volume/util/subpath" ) func TestMakeMounts(t *testing.T) { @@ -241,13 +242,14 @@ func TestMakeMounts(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { fm := &mount.FakeMounter{} + fsp := &subpath.FakeSubpath{} pod := v1.Pod{ Spec: v1.PodSpec{ HostNetwork: true, }, } - mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, nil) + mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, fsp, nil) // validate only the error if we expect an error if tc.expectErr { diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index 0b4e0e02787..844c3aa9a31 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -49,10 +49,12 @@ import ( "k8s.io/kubernetes/pkg/kubelet/server/portforward" "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume/util/subpath" ) func TestDisabledSubpath(t *testing.T) { fm := &mount.FakeMounter{} + fsp := &subpath.FakeSubpath{} pod := v1.Pod{ Spec: v1.PodSpec{ HostNetwork: true, @@ -95,7 +97,7 @@ func TestDisabledSubpath(t *testing.T) { defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, false)() for name, test := range cases { - _, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, nil) + _, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, fsp, nil) if err != nil && !test.expectError { t.Errorf("test %v failed: %v", name, err) } diff --git a/pkg/kubelet/kubelet_pods_windows_test.go b/pkg/kubelet/kubelet_pods_windows_test.go index 41a2cf6dc09..9e16cd118f3 100644 --- a/pkg/kubelet/kubelet_pods_windows_test.go +++ b/pkg/kubelet/kubelet_pods_windows_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/api/core/v1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume/util/subpath" ) func TestMakeMountsWindows(t *testing.T) { @@ -82,7 +83,8 @@ func TestMakeMountsWindows(t *testing.T) { } fm := &mount.FakeMounter{} - mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm, nil) + fsp := &subpath.FakeSubpath{} + mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm, fsp, nil) expectedMounts := []kubecontainer.Mount{ { diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 192060bda84..25d008f92bb 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -75,6 +75,7 @@ import ( _ "k8s.io/kubernetes/pkg/volume/host_path" volumetest "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util/subpath" ) func init() { @@ -176,6 +177,7 @@ func newTestKubeletWithImageList( kubelet.heartbeatClient = fakeKubeClient kubelet.os = &containertest.FakeOS{} kubelet.mounter = &mount.FakeMounter{} + kubelet.subpather = &subpath.FakeSubpath{} kubelet.hostname = testKubeletHostname kubelet.nodeName = types.NodeName(testKubeletHostname) diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index 425afebdbf2..78797d89581 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -40,6 +40,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util/subpath" ) // NewInitializedVolumePluginMgr returns a new instance of @@ -126,6 +127,10 @@ func (kvh *kubeletVolumeHost) GetCSIClient() csiclientset.Interface { return kvh.kubelet.csiClient } +func (kvh *kubeletVolumeHost) GetSubpather() subpath.Interface { + return kvh.kubelet.subpather +} + func (kvh *kubeletVolumeHost) NewWrapperMounter( volName string, spec volume.Spec, diff --git a/pkg/kubemark/BUILD b/pkg/kubemark/BUILD index 0eea6adcfe6..ced440d7bc6 100644 --- a/pkg/kubemark/BUILD +++ b/pkg/kubemark/BUILD @@ -36,6 +36,7 @@ go_library( "//pkg/volume/emptydir:go_default_library", "//pkg/volume/projected:go_default_library", "//pkg/volume/secret:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 875d84c3003..0bdc6020e5f 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -35,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/volume/emptydir" "k8s.io/kubernetes/pkg/volume/projected" "k8s.io/kubernetes/pkg/volume/secret" + "k8s.io/kubernetes/pkg/volume/util/subpath" "k8s.io/kubernetes/test/utils" "k8s.io/klog" @@ -78,6 +79,7 @@ func NewHollowKubelet( TLSOptions: nil, OOMAdjuster: oom.NewFakeOOMAdjuster(), Mounter: mount.New("" /* default mount path */), + Subpather: &subpath.FakeSubpath{}, } return &HollowKubelet{ diff --git a/pkg/util/mount/BUILD b/pkg/util/mount/BUILD index d2eae31ed0f..5b22b2b5aa6 100644 --- a/pkg/util/mount/BUILD +++ b/pkg/util/mount/BUILD @@ -80,8 +80,6 @@ go_test( "//vendor/k8s.io/utils/exec/testing:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ - "//vendor/golang.org/x/sys/unix:go_default_library", - "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], diff --git a/pkg/util/mount/exec_mount.go b/pkg/util/mount/exec_mount.go index 634189dea9b..b30f6f0218f 100644 --- a/pkg/util/mount/exec_mount.go +++ b/pkg/util/mount/exec_mount.go @@ -144,18 +144,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) { return m.wrappedMounter.EvalHostSymlinks(pathname) } -func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - return m.wrappedMounter.PrepareSafeSubpath(subPath) -} - -func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error { - return m.wrappedMounter.CleanSubPaths(podDir, volumeName) -} - -func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { - return m.wrappedMounter.SafeMakeDir(pathname, base, perm) -} - func (m *execMounter) GetMountRefs(pathname string) ([]string, error) { return m.wrappedMounter.GetMountRefs(pathname) } diff --git a/pkg/util/mount/exec_mount_unsupported.go b/pkg/util/mount/exec_mount_unsupported.go index 3be436ce440..698c136e019 100644 --- a/pkg/util/mount/exec_mount_unsupported.go +++ b/pkg/util/mount/exec_mount_unsupported.go @@ -91,18 +91,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) { return "", errors.New("not implemented") } -func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - return subPath.Path, nil, nil -} - -func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error { - return nil -} - -func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { - return nil -} - func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) { return nil, errors.New("not implemented") } diff --git a/pkg/util/mount/fake.go b/pkg/util/mount/fake.go index 8b93d3b488e..f8abc216b87 100644 --- a/pkg/util/mount/fake.go +++ b/pkg/util/mount/fake.go @@ -220,17 +220,6 @@ func (f *FakeMounter) EvalHostSymlinks(pathname string) (string, error) { return pathname, nil } -func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - return subPath.Path, nil, nil -} - -func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error { - return nil -} -func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { - return nil -} - func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) { realpath, err := filepath.EvalSymlinks(pathname) if err != nil { diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index dc108a89427..b0ad41ceffd 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -84,35 +84,12 @@ type Interface interface { // MakeDir creates a new directory. // Will operate in the host mount namespace if kubelet is running in a container MakeDir(pathname string) error - // SafeMakeDir creates subdir within given base. It makes sure that the - // created directory does not escape given base directory mis-using - // symlinks. Note that the function makes sure that it creates the directory - // somewhere under the base, nothing else. E.g. if the directory already - // exists, it may exist outside of the base due to symlinks. - // This method should be used if the directory to create is inside volume - // that's under user control. User must not be able to use symlinks to - // escape the volume to create directories somewhere else. - SafeMakeDir(subdir string, base string, perm os.FileMode) error // Will operate in the host mount namespace if kubelet is running in a container. // Error is returned on any other error than "file not found". ExistsPath(pathname string) (bool, error) // EvalHostSymlinks returns the path name after evaluating symlinks. // Will operate in the host mount namespace if kubelet is running in a container. EvalHostSymlinks(pathname string) (string, error) - // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given - // pod volume directory. - CleanSubPaths(podDir string, volumeName string) error - // PrepareSafeSubpath does everything that's necessary to prepare a subPath - // that's 1) inside given volumePath and 2) immutable after this call. - // - // newHostPath - location of prepared subPath. It should be used instead of - // hostName when running the container. - // cleanupAction - action to run when the container is running or it failed to start. - // - // CleanupAction must be called immediately after the container with given - // subpath starts. On the other hand, Interface.CleanSubPaths must be called - // when the pod finishes. - PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) // GetMountRefs finds all mount references to the path, returns a // list of paths. Path could be a mountpoint path, device or a normal // directory (for bind mount). @@ -355,15 +332,15 @@ func PathWithinBase(fullPath, basePath string) bool { if err != nil { return false } - if startsWithBackstep(rel) { + if StartsWithBackstep(rel) { // Needed to escape the base path return false } return true } -// startsWithBackstep checks if the given path starts with a backstep segment -func startsWithBackstep(rel string) bool { +// StartsWithBackstep checks if the given path starts with a backstep segment +func StartsWithBackstep(rel string) bool { // normalize to / and check for ../ return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") } diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index c66039e5e09..218757ef083 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -21,7 +21,6 @@ package mount import ( "errors" "fmt" - "io/ioutil" "os" "os/exec" "path" @@ -53,14 +52,6 @@ const ( fsckErrorsCorrected = 1 // 'fsck' found errors but exited without correcting them fsckErrorsUncorrected = 4 - - // place for subpath mounts - // TODO: pass in directory using kubelet_getters instead - containerSubPathDirectoryName = "volume-subpaths" - // syscall.Openat flags used to traverse directories not following symlinks - nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW - // flags for getting file descriptor without following the symlink - openFDFlags = unix.O_NOFOLLOW | unix.O_PATH ) // Mounter provides the default implementation of mount.Interface @@ -726,282 +717,6 @@ func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) { return false, nil } -func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - newHostPath, err = doBindSubPath(mounter, subPath) - - // There is no action when the container starts. Bind-mount will be cleaned - // when container stops by CleanSubPaths. - cleanupAction = nil - return newHostPath, cleanupAction, err -} - -// This implementation is shared between Linux and NsEnterMounter -func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) { - if !PathWithinBase(subpath.Path, subpath.VolumePath) { - return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath) - } - fd, err := doSafeOpen(subpath.Path, subpath.VolumePath) - if err != nil { - return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err) - } - return fd, nil -} - -// prepareSubpathTarget creates target for bind-mount of subpath. It returns -// "true" when the target already exists and something is mounted there. -// Given Subpath must have all paths with already resolved symlinks and with -// paths relevant to kubelet (when it runs in a container). -// This function is called also by NsEnterMounter. It works because -// /var/lib/kubelet is mounted from the host into the container with Kubelet as -// /var/lib/kubelet too. -func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) { - // Early check for already bind-mounted subpath. - bindPathTarget := getSubpathBindTarget(subpath) - notMount, err := mounter.IsNotMountPoint(bindPathTarget) - if err != nil { - if !os.IsNotExist(err) { - return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err) - } - // Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet. - notMount = true - } - if !notMount { - // It's already mounted - klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) - return true, bindPathTarget, nil - } - - // bindPathTarget is in /var/lib/kubelet and thus reachable without any - // translation even to containerized kubelet. - bindParent := filepath.Dir(bindPathTarget) - err = os.MkdirAll(bindParent, 0750) - if err != nil && !os.IsExist(err) { - return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err) - } - - t, err := os.Lstat(subpath.Path) - if err != nil { - return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err) - } - - if t.Mode()&os.ModeDir > 0 { - if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) { - return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err) - } - } else { - // "/bin/touch ". - // A file is enough for all possible targets (symlink, device, pipe, - // socket, ...), bind-mounting them into a file correctly changes type - // of the target file. - if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil { - return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err) - } - } - return false, bindPathTarget, nil -} - -func getSubpathBindTarget(subpath Subpath) string { - // containerName is DNS label, i.e. safe as a directory name. - return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex)) -} - -func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) { - // Linux, kubelet runs on the host: - // - safely open the subpath - // - bind-mount /proc//fd/ to subpath target - // User can't change /proc//fd/ to point to a bad place. - - // Evaluate all symlinks here once for all subsequent functions. - newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath) - if err != nil { - return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) - } - newPath, err := filepath.EvalSymlinks(subpath.Path) - if err != nil { - return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) - } - klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath) - subpath.VolumePath = newVolumePath - subpath.Path = newPath - - fd, err := safeOpenSubPath(mounter, subpath) - if err != nil { - return "", err - } - defer syscall.Close(fd) - - alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) - if err != nil { - return "", err - } - if alreadyMounted { - return bindPathTarget, nil - } - - success := false - defer func() { - // Cleanup subpath on error - if !success { - klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) - if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { - klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) - } - } - }() - - kubeletPid := os.Getpid() - mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) - - // Do the bind mount - options := []string{"bind"} - klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) - if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { - return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) - } - success = true - - klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) - return bindPathTarget, nil -} - -func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { - return doCleanSubPaths(mounter, podDir, volumeName) -} - -// This implementation is shared between Linux and NsEnterMounter -func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error { - // scan /var/lib/kubelet/pods//volume-subpaths//* - subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) - klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) - - containerDirs, err := ioutil.ReadDir(subPathDir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return fmt.Errorf("error reading %s: %s", subPathDir, err) - } - - for _, containerDir := range containerDirs { - if !containerDir.IsDir() { - klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) - continue - } - klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) - - // scan /var/lib/kubelet/pods//volume-subpaths///* - fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name()) - err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error { - if path == fullContainerDirPath { - // Skip top level directory - return nil - } - - // pass through errors and let doCleanSubPath handle them - if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil { - return err - } - return nil - }) - if err != nil { - return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err) - } - - // Whole container has been processed, remove its directory. - if err := os.Remove(fullContainerDirPath); err != nil { - return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err) - } - klog.V(5).Infof("Removed %s", fullContainerDirPath) - } - // Whole pod volume subpaths have been cleaned up, remove its subpath directory. - if err := os.Remove(subPathDir); err != nil { - return fmt.Errorf("error deleting %s: %s", subPathDir, err) - } - klog.V(5).Infof("Removed %s", subPathDir) - - // Remove entire subpath directory if it's the last one - podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName) - if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) { - return fmt.Errorf("error deleting %s: %s", podSubPathDir, err) - } - klog.V(5).Infof("Removed %s", podSubPathDir) - return nil -} - -// doCleanSubPath tears down the single subpath bind mount -func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error { - // process /var/lib/kubelet/pods//volume-subpaths/// - klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) - fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) - - if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil { - return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err) - } - - klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath) - return nil -} - -// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty -func cleanSubPath(mounter Interface, subpath Subpath) error { - containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) - - // Clean subdir bindmount - if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) { - return err - } - - // Recusively remove directories if empty - if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil { - return err - } - - return nil -} - -// removeEmptyDirs works backwards from endDir to baseDir and removes each directory -// if it is empty. It stops once it encounters a directory that has content -func removeEmptyDirs(baseDir, endDir string) error { - if !PathWithinBase(endDir, baseDir) { - return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) - } - - for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) { - s, err := os.Stat(curDir) - if err != nil { - if os.IsNotExist(err) { - klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) - continue - } - return fmt.Errorf("error stat %q: %v", curDir, err) - } - if !s.IsDir() { - return fmt.Errorf("path %q not a directory", curDir) - } - - err = os.Remove(curDir) - if os.IsExist(err) { - klog.V(5).Infof("Directory %q not empty, not removing", curDir) - break - } else if err != nil { - return fmt.Errorf("error removing directory %q: %v", curDir, err) - } - klog.V(5).Infof("Removed directory %q", curDir) - } - return nil -} - -func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { - realBase, err := filepath.EvalSymlinks(base) - if err != nil { - return fmt.Errorf("error resolving symlinks in %s: %s", base, err) - } - - realFullPath := filepath.Join(realBase, subdir) - - return doSafeMakeDir(realFullPath, realBase, perm) -} - func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { if _, err := os.Stat(pathname); os.IsNotExist(err) { return []string{}, nil @@ -1049,237 +764,6 @@ func getMode(pathname string) (os.FileMode, error) { return info.Mode(), nil } -// This implementation is shared between Linux and NsEnterMounter. Both pathname -// and base must be either already resolved symlinks or thet will be resolved in -// kubelet's mount namespace (in case it runs containerized). -func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { - klog.V(4).Infof("Creating directory %q within base %q", pathname, base) - - if !PathWithinBase(pathname, base) { - return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) - } - - // Quick check if the directory already exists - s, err := os.Stat(pathname) - if err == nil { - // Path exists - if s.IsDir() { - // The directory already exists. It can be outside of the parent, - // but there is no race-proof check. - klog.V(4).Infof("Directory %s already exists", pathname) - return nil - } - return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} - } - - // Find all existing directories - existingPath, toCreate, err := findExistingPrefix(base, pathname) - if err != nil { - return fmt.Errorf("error opening directory %s: %s", pathname, err) - } - // Ensure the existing directory is inside allowed base - fullExistingPath, err := filepath.EvalSymlinks(existingPath) - if err != nil { - return fmt.Errorf("error opening directory %s: %s", existingPath, err) - } - if !PathWithinBase(fullExistingPath, base) { - return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) - } - - klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) - parentFD, err := doSafeOpen(fullExistingPath, base) - if err != nil { - return fmt.Errorf("cannot open directory %s: %s", existingPath, err) - } - childFD := -1 - defer func() { - if parentFD != -1 { - if err = syscall.Close(parentFD); err != nil { - klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) - } - } - if childFD != -1 { - if err = syscall.Close(childFD); err != nil { - klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) - } - } - }() - - currentPath := fullExistingPath - // create the directories one by one, making sure nobody can change - // created directory into symlink. - for _, dir := range toCreate { - currentPath = filepath.Join(currentPath, dir) - klog.V(4).Infof("Creating %s", dir) - err = syscall.Mkdirat(parentFD, currentPath, uint32(perm)) - if err != nil { - return fmt.Errorf("cannot create directory %s: %s", currentPath, err) - } - // Dive into the created directory - childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0) - if err != nil { - return fmt.Errorf("cannot open %s: %s", currentPath, err) - } - // We can be sure that childFD is safe to use. It could be changed - // by user after Mkdirat() and before Openat(), however: - // - it could not be changed to symlink - we use nofollowFlags - // - it could be changed to a file (or device, pipe, socket, ...) - // but either subsequent Mkdirat() fails or we mount this file - // to user's container. Security is no violated in both cases - // and user either gets error or the file that it can already access. - - if err = syscall.Close(parentFD); err != nil { - klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) - } - parentFD = childFD - childFD = -1 - } - - // Everything was created. mkdirat(..., perm) above was affected by current - // umask and we must apply the right permissions to the last directory - // (that's the one that will be available to the container as subpath) - // so user can read/write it. This is the behavior of previous code. - // TODO: chmod all created directories, not just the last one. - // parentFD is the last created directory. - - // Translate perm (os.FileMode) to uint32 that fchmod() expects - kernelPerm := uint32(perm & os.ModePerm) - if perm&os.ModeSetgid > 0 { - kernelPerm |= syscall.S_ISGID - } - if perm&os.ModeSetuid > 0 { - kernelPerm |= syscall.S_ISUID - } - if perm&os.ModeSticky > 0 { - kernelPerm |= syscall.S_ISVTX - } - if err = syscall.Fchmod(parentFD, kernelPerm); err != nil { - return fmt.Errorf("chmod %q failed: %s", currentPath, err) - } - return nil -} - -// findExistingPrefix finds prefix of pathname that exists. In addition, it -// returns list of remaining directories that don't exist yet. -func findExistingPrefix(base, pathname string) (string, []string, error) { - rel, err := filepath.Rel(base, pathname) - if err != nil { - return base, nil, err - } - dirs := strings.Split(rel, string(filepath.Separator)) - - // Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks. - // This should be faster than looping through all dirs and calling os.Stat() - // on each of them, as the symlinks are resolved only once with OpenAt(). - currentPath := base - fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0) - if err != nil { - return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err) - } - defer func() { - if err = syscall.Close(fd); err != nil { - klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) - } - }() - for i, dir := range dirs { - // Using O_PATH here will prevent hangs in case user replaces directory with - // fifo - childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0) - if err != nil { - if os.IsNotExist(err) { - return currentPath, dirs[i:], nil - } - return base, nil, err - } - if err = syscall.Close(fd); err != nil { - klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) - } - fd = childFD - currentPath = filepath.Join(currentPath, dir) - } - return pathname, []string{}, nil -} - -// This implementation is shared between Linux and NsEnterMounter -// Open path and return its fd. -// Symlinks are disallowed (pathname must already resolve symlinks), -// and the path must be within the base directory. -func doSafeOpen(pathname string, base string) (int, error) { - pathname = filepath.Clean(pathname) - base = filepath.Clean(base) - - // Calculate segments to follow - subpath, err := filepath.Rel(base, pathname) - if err != nil { - return -1, err - } - segments := strings.Split(subpath, string(filepath.Separator)) - - // Assumption: base is the only directory that we have under control. - // Base dir is not allowed to be a symlink. - parentFD, err := syscall.Open(base, nofollowFlags, 0) - if err != nil { - return -1, fmt.Errorf("cannot open directory %s: %s", base, err) - } - defer func() { - if parentFD != -1 { - if err = syscall.Close(parentFD); err != nil { - klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) - } - } - }() - - childFD := -1 - defer func() { - if childFD != -1 { - if err = syscall.Close(childFD); err != nil { - klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) - } - } - }() - - currentPath := base - - // Follow the segments one by one using openat() to make - // sure the user cannot change already existing directories into symlinks. - for _, seg := range segments { - currentPath = filepath.Join(currentPath, seg) - if !PathWithinBase(currentPath, base) { - return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) - } - - klog.V(5).Infof("Opening path %s", currentPath) - childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0) - if err != nil { - return -1, fmt.Errorf("cannot open %s: %s", currentPath, err) - } - - var deviceStat unix.Stat_t - err := unix.Fstat(childFD, &deviceStat) - if err != nil { - return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err) - } - fileFmt := deviceStat.Mode & syscall.S_IFMT - if fileFmt == syscall.S_IFLNK { - return -1, fmt.Errorf("Unexpected symlink found %s", currentPath) - } - - // Close parentFD - if err = syscall.Close(parentFD); err != nil { - return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err) - } - // Set child to new parent - parentFD = childFD - childFD = -1 - } - - // We made it to the end, return this fd, don't close it - finalFD := parentFD - parentFD = -1 - - return finalFD, nil -} - // searchMountPoints finds all mount references to the source, returns a list of // mountpoints. // This function assumes source cannot be device. diff --git a/pkg/util/mount/mount_linux_test.go b/pkg/util/mount/mount_linux_test.go index e2669dd0f7b..ee2769a8270 100644 --- a/pkg/util/mount/mount_linux_test.go +++ b/pkg/util/mount/mount_linux_test.go @@ -25,14 +25,10 @@ import ( "os" "path/filepath" "reflect" - "strconv" "strings" - "syscall" "testing" "k8s.io/utils/exec" - - "k8s.io/klog" ) func TestReadProcMountsFrom(t *testing.T) { @@ -424,822 +420,6 @@ func TestPathWithinBase(t *testing.T) { } } -func TestSafeMakeDir(t *testing.T) { - defaultPerm := os.FileMode(0750) + os.ModeDir - tests := []struct { - name string - // Function that prepares directory structure for the test under given - // base. - prepare func(base string) error - path string - checkPath string - perm os.FileMode - expectError bool - }{ - { - "directory-does-not-exist", - func(base string) error { - return nil - }, - "test/directory", - "test/directory", - defaultPerm, - false, - }, - { - "directory-with-sgid", - func(base string) error { - return nil - }, - "test/directory", - "test/directory", - os.FileMode(0777) + os.ModeDir + os.ModeSetgid, - false, - }, - { - "directory-with-suid", - func(base string) error { - return nil - }, - "test/directory", - "test/directory", - os.FileMode(0777) + os.ModeDir + os.ModeSetuid, - false, - }, - { - "directory-with-sticky-bit", - func(base string) error { - return nil - }, - "test/directory", - "test/directory", - os.FileMode(0777) + os.ModeDir + os.ModeSticky, - false, - }, - { - "directory-exists", - func(base string) error { - return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) - }, - "test/directory", - "test/directory", - defaultPerm, - false, - }, - { - "create-base", - func(base string) error { - return nil - }, - "", - "", - defaultPerm, - false, - }, - { - "escape-base-using-dots", - func(base string) error { - return nil - }, - "..", - "", - defaultPerm, - true, - }, - { - "escape-base-using-dots-2", - func(base string) error { - return nil - }, - "test/../../..", - "", - defaultPerm, - true, - }, - { - "follow-symlinks", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil { - return err - } - return os.Symlink("destination", filepath.Join(base, "test")) - }, - "test/directory", - "destination/directory", - defaultPerm, - false, - }, - { - "follow-symlink-loop", - func(base string) error { - return os.Symlink("test", filepath.Join(base, "test")) - }, - "test/directory", - "", - defaultPerm, - true, - }, - { - "follow-symlink-multiple follow", - func(base string) error { - /* test1/dir points to test2 and test2/dir points to test1 */ - if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil { - return err - } - if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil { - return err - } - if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil { - return err - } - if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil { - return err - } - return nil - }, - "test1/dir/dir/dir/dir/dir/dir/dir/foo", - "test2/foo", - defaultPerm, - false, - }, - { - "danglink-symlink", - func(base string) error { - return os.Symlink("non-existing", filepath.Join(base, "test")) - }, - "test/directory", - "", - defaultPerm, - true, - }, - { - "non-directory", - func(base string) error { - return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) - }, - "test/directory", - "", - defaultPerm, - true, - }, - { - "non-directory-final", - func(base string) error { - return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) - }, - "test", - "", - defaultPerm, - true, - }, - { - "escape-with-relative-symlink", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { - return err - } - if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil { - return err - } - return os.Symlink("../exists", filepath.Join(base, "dir/test")) - }, - "dir/test", - "", - defaultPerm, - false, - }, - { - "escape-with-relative-symlink-not-exists", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { - return err - } - return os.Symlink("../not-exists", filepath.Join(base, "dir/test")) - }, - "dir/test", - "", - defaultPerm, - true, - }, - { - "escape-with-symlink", - func(base string) error { - return os.Symlink("/", filepath.Join(base, "test")) - }, - "test/directory", - "", - defaultPerm, - true, - }, - } - - for i := range tests { - test := tests[i] - t.Run(test.name, func(t *testing.T) { - base, err := ioutil.TempDir("", "safe-make-dir-"+test.name+"-") - if err != nil { - t.Fatalf(err.Error()) - } - defer os.RemoveAll(base) - test.prepare(base) - pathToCreate := filepath.Join(base, test.path) - err = doSafeMakeDir(pathToCreate, base, test.perm) - if err != nil && !test.expectError { - t.Fatal(err) - } - if err != nil { - t.Logf("got error: %s", err) - } - if err == nil && test.expectError { - t.Fatalf("expected error, got none") - } - - if test.checkPath != "" { - st, err := os.Stat(filepath.Join(base, test.checkPath)) - if err != nil { - t.Fatalf("cannot read path %s", test.checkPath) - } - actualMode := st.Mode() - if actualMode != test.perm { - if actualMode^test.perm == os.ModeSetgid && test.perm&os.ModeSetgid == 0 { - // when TMPDIR is a kubernetes emptydir, the sticky gid bit is set due to fsgroup - t.Logf("masking bit from %o", actualMode) - } else { - t.Errorf("expected permissions %o, got %o (%b)", test.perm, actualMode, test.perm^actualMode) - } - } - } - }) - } -} - -func TestRemoveEmptyDirs(t *testing.T) { - defaultPerm := os.FileMode(0750) - tests := []struct { - name string - // Function that prepares directory structure for the test under given - // base. - prepare func(base string) error - // Function that validates directory structure after the test - validate func(base string) error - baseDir string - endDir string - expectError bool - }{ - { - name: "all-empty", - prepare: func(base string) error { - return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm) - }, - validate: func(base string) error { - return validateDirEmpty(filepath.Join(base, "a")) - }, - baseDir: "a", - endDir: "a/b/c", - expectError: false, - }, - { - name: "dir-not-empty", - prepare: func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm); err != nil { - return err - } - return os.Mkdir(filepath.Join(base, "a/b/d"), defaultPerm) - }, - validate: func(base string) error { - if err := validateDirNotExists(filepath.Join(base, "a/b/c")); err != nil { - return err - } - return validateDirExists(filepath.Join(base, "a/b")) - }, - baseDir: "a", - endDir: "a/b/c", - expectError: false, - }, - { - name: "path-not-within-base", - prepare: func(base string) error { - return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm) - }, - validate: func(base string) error { - return validateDirExists(filepath.Join(base, "a")) - }, - baseDir: "a", - endDir: "b/c", - expectError: true, - }, - { - name: "path-already-deleted", - prepare: func(base string) error { - return nil - }, - validate: func(base string) error { - return nil - }, - baseDir: "a", - endDir: "a/b/c", - expectError: false, - }, - { - name: "path-not-dir", - prepare: func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "a/b"), defaultPerm); err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(base, "a/b", "c"), []byte{}, defaultPerm) - }, - validate: func(base string) error { - if err := validateDirExists(filepath.Join(base, "a/b")); err != nil { - return err - } - return validateFileExists(filepath.Join(base, "a/b/c")) - }, - baseDir: "a", - endDir: "a/b/c", - expectError: true, - }, - } - - for _, test := range tests { - klog.V(4).Infof("test %q", test.name) - base, err := ioutil.TempDir("", "remove-empty-dirs-"+test.name+"-") - if err != nil { - t.Fatalf(err.Error()) - } - if err = test.prepare(base); err != nil { - os.RemoveAll(base) - t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) - } - - err = removeEmptyDirs(filepath.Join(base, test.baseDir), filepath.Join(base, test.endDir)) - if err != nil && !test.expectError { - t.Errorf("test %q failed: %v", test.name, err) - } - if err == nil && test.expectError { - t.Errorf("test %q failed: expected error, got success", test.name) - } - - if err = test.validate(base); err != nil { - t.Errorf("test %q failed validation: %v", test.name, err) - } - - os.RemoveAll(base) - } -} - -func TestCleanSubPaths(t *testing.T) { - defaultPerm := os.FileMode(0750) - testVol := "vol1" - - tests := []struct { - name string - // Function that prepares directory structure for the test under given - // base. - prepare func(base string) ([]MountPoint, error) - // Function that validates directory structure after the test - validate func(base string) error - expectError bool - }{ - { - name: "not-exists", - prepare: func(base string) ([]MountPoint, error) { - return nil, nil - }, - validate: func(base string) error { - return nil - }, - expectError: false, - }, - { - name: "subpath-not-mount", - prepare: func(base string) ([]MountPoint, error) { - return nil, os.MkdirAll(filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0"), defaultPerm) - }, - validate: func(base string) error { - return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) - }, - expectError: false, - }, - { - name: "subpath-file", - prepare: func(base string) ([]MountPoint, error) { - path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1") - if err := os.MkdirAll(path, defaultPerm); err != nil { - return nil, err - } - return nil, ioutil.WriteFile(filepath.Join(path, "0"), []byte{}, defaultPerm) - }, - validate: func(base string) error { - return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) - }, - expectError: false, - }, - { - name: "subpath-container-not-dir", - prepare: func(base string) ([]MountPoint, error) { - path := filepath.Join(base, containerSubPathDirectoryName, testVol) - if err := os.MkdirAll(path, defaultPerm); err != nil { - return nil, err - } - return nil, ioutil.WriteFile(filepath.Join(path, "container1"), []byte{}, defaultPerm) - }, - validate: func(base string) error { - return validateDirExists(filepath.Join(base, containerSubPathDirectoryName, testVol)) - }, - expectError: true, - }, - { - name: "subpath-multiple-container-not-dir", - prepare: func(base string) ([]MountPoint, error) { - path := filepath.Join(base, containerSubPathDirectoryName, testVol) - if err := os.MkdirAll(filepath.Join(path, "container1"), defaultPerm); err != nil { - return nil, err - } - return nil, ioutil.WriteFile(filepath.Join(path, "container2"), []byte{}, defaultPerm) - }, - validate: func(base string) error { - path := filepath.Join(base, containerSubPathDirectoryName, testVol) - if err := validateDirNotExists(filepath.Join(path, "container1")); err != nil { - return err - } - return validateFileExists(filepath.Join(path, "container2")) - }, - expectError: true, - }, - { - name: "subpath-mount", - prepare: func(base string) ([]MountPoint, error) { - path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") - if err := os.MkdirAll(path, defaultPerm); err != nil { - return nil, err - } - mounts := []MountPoint{{Device: "/dev/sdb", Path: path}} - return mounts, nil - }, - validate: func(base string) error { - return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) - }, - }, - { - name: "subpath-mount-multiple", - prepare: func(base string) ([]MountPoint, error) { - path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") - path2 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "1") - path3 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container2", "1") - if err := os.MkdirAll(path, defaultPerm); err != nil { - return nil, err - } - if err := os.MkdirAll(path2, defaultPerm); err != nil { - return nil, err - } - if err := os.MkdirAll(path3, defaultPerm); err != nil { - return nil, err - } - mounts := []MountPoint{ - {Device: "/dev/sdb", Path: path}, - {Device: "/dev/sdb", Path: path3}, - } - return mounts, nil - }, - validate: func(base string) error { - return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) - }, - }, - { - name: "subpath-mount-multiple-vols", - prepare: func(base string) ([]MountPoint, error) { - path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") - path2 := filepath.Join(base, containerSubPathDirectoryName, "vol2", "container1", "1") - if err := os.MkdirAll(path, defaultPerm); err != nil { - return nil, err - } - if err := os.MkdirAll(path2, defaultPerm); err != nil { - return nil, err - } - mounts := []MountPoint{ - {Device: "/dev/sdb", Path: path}, - } - return mounts, nil - }, - validate: func(base string) error { - baseSubdir := filepath.Join(base, containerSubPathDirectoryName) - if err := validateDirNotExists(filepath.Join(baseSubdir, testVol)); err != nil { - return err - } - return validateDirExists(baseSubdir) - }, - }, - } - - for _, test := range tests { - klog.V(4).Infof("test %q", test.name) - base, err := ioutil.TempDir("", "clean-subpaths-"+test.name+"-") - if err != nil { - t.Fatalf(err.Error()) - } - mounts, err := test.prepare(base) - if err != nil { - os.RemoveAll(base) - t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) - } - - fm := &FakeMounter{MountPoints: mounts} - - err = doCleanSubPaths(fm, base, testVol) - if err != nil && !test.expectError { - t.Errorf("test %q failed: %v", test.name, err) - } - if err == nil && test.expectError { - t.Errorf("test %q failed: expected error, got success", test.name) - } - if err = test.validate(base); err != nil { - t.Errorf("test %q failed validation: %v", test.name, err) - } - - os.RemoveAll(base) - } -} - -var ( - testVol = "vol1" - testPod = "pod0" - testContainer = "container0" - testSubpath = 1 -) - -func setupFakeMounter(testMounts []string) *FakeMounter { - mounts := []MountPoint{} - for _, mountPoint := range testMounts { - mounts = append(mounts, MountPoint{Device: "/foo", Path: mountPoint}) - } - return &FakeMounter{MountPoints: mounts} -} - -func getTestPaths(base string) (string, string) { - return filepath.Join(base, testVol), - filepath.Join(base, testPod, containerSubPathDirectoryName, testVol, testContainer, strconv.Itoa(testSubpath)) -} - -func TestBindSubPath(t *testing.T) { - defaultPerm := os.FileMode(0750) - - tests := []struct { - name string - // Function that prepares directory structure for the test under given - // base. - prepare func(base string) ([]string, string, string, error) - expectError bool - }{ - { - name: "subpath-dir", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpath := filepath.Join(volpath, "dir0") - return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm) - }, - expectError: false, - }, - { - name: "subpath-dir-symlink", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpath := filepath.Join(volpath, "dir0") - if err := os.MkdirAll(subpath, defaultPerm); err != nil { - return nil, "", "", err - } - subpathLink := filepath.Join(volpath, "dirLink") - return nil, volpath, subpath, os.Symlink(subpath, subpathLink) - }, - expectError: false, - }, - { - name: "subpath-file", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpath := filepath.Join(volpath, "file0") - if err := os.MkdirAll(volpath, defaultPerm); err != nil { - return nil, "", "", err - } - return nil, volpath, subpath, ioutil.WriteFile(subpath, []byte{}, defaultPerm) - }, - expectError: false, - }, - { - name: "subpath-not-exists", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpath := filepath.Join(volpath, "file0") - return nil, volpath, subpath, nil - }, - expectError: true, - }, - { - name: "subpath-outside", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpath := filepath.Join(volpath, "dir0") - if err := os.MkdirAll(volpath, defaultPerm); err != nil { - return nil, "", "", err - } - return nil, volpath, subpath, os.Symlink(base, subpath) - }, - expectError: true, - }, - { - name: "subpath-symlink-child-outside", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpathDir := filepath.Join(volpath, "dir0") - subpath := filepath.Join(subpathDir, "child0") - if err := os.MkdirAll(subpathDir, defaultPerm); err != nil { - return nil, "", "", err - } - return nil, volpath, subpath, os.Symlink(base, subpath) - }, - expectError: true, - }, - { - name: "subpath-child-outside-exists", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpathDir := filepath.Join(volpath, "dir0") - child := filepath.Join(base, "child0") - subpath := filepath.Join(subpathDir, "child0") - if err := os.MkdirAll(volpath, defaultPerm); err != nil { - return nil, "", "", err - } - // touch file outside - if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil { - return nil, "", "", err - } - - // create symlink for subpath dir - return nil, volpath, subpath, os.Symlink(base, subpathDir) - }, - expectError: true, - }, - { - name: "subpath-child-outside-not-exists", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpathDir := filepath.Join(volpath, "dir0") - subpath := filepath.Join(subpathDir, "child0") - if err := os.MkdirAll(volpath, defaultPerm); err != nil { - return nil, "", "", err - } - // create symlink for subpath dir - return nil, volpath, subpath, os.Symlink(base, subpathDir) - }, - expectError: true, - }, - { - name: "subpath-child-outside-exists-middle-dir-symlink", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpathDir := filepath.Join(volpath, "dir0") - symlinkDir := filepath.Join(subpathDir, "linkDir0") - child := filepath.Join(base, "child0") - subpath := filepath.Join(symlinkDir, "child0") - if err := os.MkdirAll(subpathDir, defaultPerm); err != nil { - return nil, "", "", err - } - // touch file outside - if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil { - return nil, "", "", err - } - - // create symlink for middle dir - return nil, volpath, subpath, os.Symlink(base, symlinkDir) - }, - expectError: true, - }, - { - name: "subpath-backstepping", - prepare: func(base string) ([]string, string, string, error) { - volpath, _ := getTestPaths(base) - subpath := filepath.Join(volpath, "dir0") - symlinkBase := filepath.Join(volpath, "..") - if err := os.MkdirAll(volpath, defaultPerm); err != nil { - return nil, "", "", err - } - - // create symlink for subpath - return nil, volpath, subpath, os.Symlink(symlinkBase, subpath) - }, - expectError: true, - }, - { - name: "subpath-mountdir-already-exists", - prepare: func(base string) ([]string, string, string, error) { - volpath, subpathMount := getTestPaths(base) - if err := os.MkdirAll(subpathMount, defaultPerm); err != nil { - return nil, "", "", err - } - - subpath := filepath.Join(volpath, "dir0") - return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm) - }, - expectError: false, - }, - { - name: "subpath-mount-already-exists", - prepare: func(base string) ([]string, string, string, error) { - volpath, subpathMount := getTestPaths(base) - mounts := []string{subpathMount} - if err := os.MkdirAll(subpathMount, defaultPerm); err != nil { - return nil, "", "", err - } - - subpath := filepath.Join(volpath, "dir0") - return mounts, volpath, subpath, os.MkdirAll(subpath, defaultPerm) - }, - expectError: false, - }, - { - name: "mount-unix-socket", - prepare: func(base string) ([]string, string, string, error) { - volpath, subpathMount := getTestPaths(base) - mounts := []string{subpathMount} - if err := os.MkdirAll(volpath, defaultPerm); err != nil { - return nil, "", "", err - } - - socketFile, socketCreateError := createSocketFile(volpath) - - return mounts, volpath, socketFile, socketCreateError - }, - expectError: false, - }, - { - name: "subpath-mounting-fifo", - prepare: func(base string) ([]string, string, string, error) { - volpath, subpathMount := getTestPaths(base) - mounts := []string{subpathMount} - if err := os.MkdirAll(volpath, defaultPerm); err != nil { - return nil, "", "", err - } - - testFifo := filepath.Join(volpath, "mount_test.fifo") - err := syscall.Mkfifo(testFifo, 0) - return mounts, volpath, testFifo, err - }, - expectError: false, - }, - } - - for _, test := range tests { - klog.V(4).Infof("test %q", test.name) - base, err := ioutil.TempDir("", "bind-subpath-"+test.name+"-") - if err != nil { - t.Fatalf(err.Error()) - } - - mounts, volPath, subPath, err := test.prepare(base) - if err != nil { - os.RemoveAll(base) - t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) - } - - fm := setupFakeMounter(mounts) - - subpath := Subpath{ - VolumeMountIndex: testSubpath, - Path: subPath, - VolumeName: testVol, - VolumePath: volPath, - PodDir: filepath.Join(base, "pod0"), - ContainerName: testContainer, - } - - _, subpathMount := getTestPaths(base) - bindPathTarget, err := doBindSubPath(fm, subpath) - if test.expectError { - if err == nil { - t.Errorf("test %q failed: expected error, got success", test.name) - } - if bindPathTarget != "" { - t.Errorf("test %q failed: expected empty bindPathTarget, got %v", test.name, bindPathTarget) - } - if err = validateDirNotExists(subpathMount); err != nil { - t.Errorf("test %q failed: %v", test.name, err) - } - } - if !test.expectError { - if err != nil { - t.Errorf("test %q failed: %v", test.name, err) - } - if bindPathTarget != subpathMount { - t.Errorf("test %q failed: expected bindPathTarget %v, got %v", test.name, subpathMount, bindPathTarget) - } - if err = validateFileExists(subpathMount); err != nil { - t.Errorf("test %q failed: %v", test.name, err) - } - } - - os.RemoveAll(base) - } -} - func TestParseMountInfo(t *testing.T) { info := `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered @@ -1461,189 +641,6 @@ func TestGetSELinuxSupport(t *testing.T) { } } -func TestSafeOpen(t *testing.T) { - defaultPerm := os.FileMode(0750) - - tests := []struct { - name string - // Function that prepares directory structure for the test under given - // base. - prepare func(base string) error - path string - expectError bool - }{ - { - "directory-does-not-exist", - func(base string) error { - return nil - }, - "test/directory", - true, - }, - { - "directory-exists", - func(base string) error { - return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) - }, - "test/directory", - false, - }, - { - "escape-base-using-dots", - func(base string) error { - return nil - }, - "..", - true, - }, - { - "escape-base-using-dots-2", - func(base string) error { - return os.MkdirAll(filepath.Join(base, "test"), 0750) - }, - "test/../../..", - true, - }, - { - "symlink", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil { - return err - } - return os.Symlink("destination", filepath.Join(base, "test")) - }, - "test", - true, - }, - { - "symlink-nested", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "dir1/dir2"), defaultPerm); err != nil { - return err - } - return os.Symlink("dir1", filepath.Join(base, "dir1/dir2/test")) - }, - "test", - true, - }, - { - "symlink-loop", - func(base string) error { - return os.Symlink("test", filepath.Join(base, "test")) - }, - "test", - true, - }, - { - "symlink-not-exists", - func(base string) error { - return os.Symlink("non-existing", filepath.Join(base, "test")) - }, - "test", - true, - }, - { - "non-directory", - func(base string) error { - return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) - }, - "test/directory", - true, - }, - { - "non-directory-final", - func(base string) error { - return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) - }, - "test", - false, - }, - { - "escape-with-relative-symlink", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { - return err - } - if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil { - return err - } - return os.Symlink("../exists", filepath.Join(base, "dir/test")) - }, - "dir/test", - true, - }, - { - "escape-with-relative-symlink-not-exists", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { - return err - } - return os.Symlink("../not-exists", filepath.Join(base, "dir/test")) - }, - "dir/test", - true, - }, - { - "escape-with-symlink", - func(base string) error { - return os.Symlink("/", filepath.Join(base, "test")) - }, - "test", - true, - }, - { - "mount-unix-socket", - func(base string) error { - socketFile, socketError := createSocketFile(base) - - if socketError != nil { - return fmt.Errorf("Error preparing socket file %s with %v", socketFile, socketError) - } - return nil - }, - "mt.sock", - false, - }, - { - "mounting-unix-socket-in-middle", - func(base string) error { - testSocketFile, socketError := createSocketFile(base) - - if socketError != nil { - return fmt.Errorf("Error preparing socket file %s with %v", testSocketFile, socketError) - } - return nil - }, - "mt.sock/bar", - true, - }, - } - - for _, test := range tests { - klog.V(4).Infof("test %q", test.name) - base, err := ioutil.TempDir("", "safe-open-"+test.name+"-") - if err != nil { - t.Fatalf(err.Error()) - } - - test.prepare(base) - pathToCreate := filepath.Join(base, test.path) - fd, err := doSafeOpen(pathToCreate, base) - if err != nil && !test.expectError { - t.Errorf("test %q: %s", test.name, err) - } - if err != nil { - klog.Infof("got error: %s", err) - } - if err == nil && test.expectError { - t.Errorf("test %q: expected error, got none", test.name) - } - - syscall.Close(fd) - os.RemoveAll(base) - } -} - func createSocketFile(socketDir string) (string, error) { testSocketFile := filepath.Join(socketDir, "mt.sock") @@ -1663,139 +660,6 @@ func createSocketFile(socketDir string) (string, error) { return testSocketFile, socketCreateError } -func TestFindExistingPrefix(t *testing.T) { - defaultPerm := os.FileMode(0750) - tests := []struct { - name string - // Function that prepares directory structure for the test under given - // base. - prepare func(base string) error - path string - expectedPath string - expectedDirs []string - expectError bool - }{ - { - "directory-does-not-exist", - func(base string) error { - return nil - }, - "directory", - "", - []string{"directory"}, - false, - }, - { - "directory-exists", - func(base string) error { - return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) - }, - "test/directory", - "test/directory", - []string{}, - false, - }, - { - "follow-symlinks", - func(base string) error { - if err := os.MkdirAll(filepath.Join(base, "destination/directory"), defaultPerm); err != nil { - return err - } - return os.Symlink("destination", filepath.Join(base, "test")) - }, - "test/directory", - "test/directory", - []string{}, - false, - }, - { - "follow-symlink-loop", - func(base string) error { - return os.Symlink("test", filepath.Join(base, "test")) - }, - "test/directory", - "", - nil, - true, - }, - { - "follow-symlink-multiple follow", - func(base string) error { - /* test1/dir points to test2 and test2/dir points to test1 */ - if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil { - return err - } - if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil { - return err - } - if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil { - return err - } - if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil { - return err - } - return nil - }, - "test1/dir/dir/foo/bar", - "test1/dir/dir", - []string{"foo", "bar"}, - false, - }, - { - "danglink-symlink", - func(base string) error { - return os.Symlink("non-existing", filepath.Join(base, "test")) - }, - // OS returns IsNotExist error both for dangling symlink and for - // non-existing directory. - "test/directory", - "", - []string{"test", "directory"}, - false, - }, - { - "with-fifo-in-middle", - func(base string) error { - testFifo := filepath.Join(base, "mount_test.fifo") - return syscall.Mkfifo(testFifo, 0) - }, - "mount_test.fifo/directory", - "", - nil, - true, - }, - } - - for _, test := range tests { - klog.V(4).Infof("test %q", test.name) - base, err := ioutil.TempDir("", "find-prefix-"+test.name+"-") - if err != nil { - t.Fatalf(err.Error()) - } - test.prepare(base) - path := filepath.Join(base, test.path) - existingPath, dirs, err := findExistingPrefix(base, path) - if err != nil && !test.expectError { - t.Errorf("test %q: %s", test.name, err) - } - if err != nil { - klog.Infof("got error: %s", err) - } - if err == nil && test.expectError { - t.Errorf("test %q: expected error, got none", test.name) - } - - fullExpectedPath := filepath.Join(base, test.expectedPath) - if existingPath != fullExpectedPath { - t.Errorf("test %q: expected path %q, got %q", test.name, fullExpectedPath, existingPath) - } - if !reflect.DeepEqual(dirs, test.expectedDirs) { - t.Errorf("test %q: expected dirs %v, got %v", test.name, test.expectedDirs, dirs) - } - os.RemoveAll(base) - } -} - func TestGetFileType(t *testing.T) { mounter := Mounter{"fake/path", false} diff --git a/pkg/util/mount/mount_unsupported.go b/pkg/util/mount/mount_unsupported.go index 918d58cd247..58f1acc7299 100644 --- a/pkg/util/mount/mount_unsupported.go +++ b/pkg/util/mount/mount_unsupported.go @@ -110,18 +110,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { return "", unsupportedErr } -func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - return subPath.Path, nil, unsupportedErr -} - -func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { - return unsupportedErr -} - -func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { - return unsupportedErr -} - func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { return nil, errors.New("not implemented") } diff --git a/pkg/util/mount/mount_windows.go b/pkg/util/mount/mount_windows.go index 917d0ce5114..cbc065ec9ae 100644 --- a/pkg/util/mount/mount_windows.go +++ b/pkg/util/mount/mount_windows.go @@ -26,7 +26,6 @@ import ( "path/filepath" "strconv" "strings" - "syscall" "k8s.io/klog" @@ -279,123 +278,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { return filepath.EvalSymlinks(pathname) } -// check whether hostPath is within volume path -// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started -func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) { - if len(volumePath) == 0 || len(hostPath) == 0 { - return []uintptr{}, nil - } - - finalSubPath, err := filepath.EvalSymlinks(hostPath) - if err != nil { - return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err) - } - finalVolumePath, err := filepath.EvalSymlinks(volumePath) - if err != nil { - return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err) - } - - return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath) -} - -// lock all intermediate subPath directories and check they are all within volumePath -// volumePath & subPath should not contain any symlink, otherwise it will return error -func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) { - if len(volumePath) == 0 || len(subPath) == 0 { - return []uintptr{}, nil - } - - // get relative path to volumePath - relSubPath, err := filepath.Rel(volumePath, subPath) - if err != nil { - return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err) - } - if startsWithBackstep(relSubPath) { - return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath) - } - - if relSubPath == "." { - // volumePath and subPath are equal - return []uintptr{}, nil - } - - fileHandles := []uintptr{} - var errorResult error - - currentFullPath := volumePath - dirs := strings.Split(relSubPath, string(os.PathSeparator)) - for _, dir := range dirs { - // lock intermediate subPath directory first - currentFullPath = filepath.Join(currentFullPath, dir) - handle, err := lockPath(currentFullPath) - if err != nil { - errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err) - break - } - fileHandles = append(fileHandles, handle) - - // make sure intermediate subPath directory does not contain symlink any more - stat, err := os.Lstat(currentFullPath) - if err != nil { - errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err) - break - } - if stat.Mode()&os.ModeSymlink != 0 { - errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath) - break - } - - if !PathWithinBase(currentFullPath, volumePath) { - errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath) - break - } - } - - return fileHandles, errorResult -} - -// unlockPath unlock directories -func unlockPath(fileHandles []uintptr) { - if fileHandles != nil { - for _, handle := range fileHandles { - syscall.CloseHandle(syscall.Handle(handle)) - } - } -} - -// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path -func lockPath(path string) (uintptr, error) { - if len(path) == 0 { - return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND - } - pathp, err := syscall.UTF16PtrFromString(path) - if err != nil { - return uintptr(syscall.InvalidHandle), err - } - access := uint32(syscall.GENERIC_READ) - sharemode := uint32(syscall.FILE_SHARE_READ) - createmode := uint32(syscall.OPEN_EXISTING) - flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT) - fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0) - return uintptr(fd), err -} - -// Lock all directories in subPath and check they're not symlinks. -func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path) - - // Unlock the directories when the container starts - cleanupAction = func() { - unlockPath(handles) - } - return subPath.Path, cleanupAction, err -} - -// No bind-mounts for subpaths are necessary on Windows -func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { - return nil -} - func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { // Try to mount the disk klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) @@ -522,126 +404,3 @@ func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { } return info.Mode(), nil } - -// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. -func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { - realBase, err := filepath.EvalSymlinks(base) - if err != nil { - return fmt.Errorf("error resolving symlinks in %s: %s", base, err) - } - - realFullPath := filepath.Join(realBase, subdir) - return doSafeMakeDir(realFullPath, realBase, perm) -} - -func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { - klog.V(4).Infof("Creating directory %q within base %q", pathname, base) - - if !PathWithinBase(pathname, base) { - return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) - } - - // Quick check if the directory already exists - s, err := os.Stat(pathname) - if err == nil { - // Path exists - if s.IsDir() { - // The directory already exists. It can be outside of the parent, - // but there is no race-proof check. - klog.V(4).Infof("Directory %s already exists", pathname) - return nil - } - return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} - } - - // Find all existing directories - existingPath, toCreate, err := findExistingPrefix(base, pathname) - if err != nil { - return fmt.Errorf("error opening directory %s: %s", pathname, err) - } - if len(toCreate) == 0 { - return nil - } - - // Ensure the existing directory is inside allowed base - fullExistingPath, err := filepath.EvalSymlinks(existingPath) - if err != nil { - return fmt.Errorf("error opening existing directory %s: %s", existingPath, err) - } - fullBasePath, err := filepath.EvalSymlinks(base) - if err != nil { - return fmt.Errorf("cannot read link %s: %s", base, err) - } - if !PathWithinBase(fullExistingPath, fullBasePath) { - return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) - } - - // lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom) - fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath) - defer unlockPath(fileHandles) - if err != nil { - return err - } - - klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) - currentPath := fullExistingPath - // create the directories one by one, making sure nobody can change - // created directory into symlink by lock that directory immediately - for _, dir := range toCreate { - currentPath = filepath.Join(currentPath, dir) - klog.V(4).Infof("Creating %s", dir) - if err := os.Mkdir(currentPath, perm); err != nil { - return fmt.Errorf("cannot create directory %s: %s", currentPath, err) - } - handle, err := lockPath(currentPath) - if err != nil { - return fmt.Errorf("cannot lock path %s: %s", currentPath, err) - } - defer syscall.CloseHandle(syscall.Handle(handle)) - // make sure newly created directory does not contain symlink after lock - stat, err := os.Lstat(currentPath) - if err != nil { - return fmt.Errorf("Lstat(%q) error: %v", currentPath, err) - } - if stat.Mode()&os.ModeSymlink != 0 { - return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath) - } - } - - return nil -} - -// findExistingPrefix finds prefix of pathname that exists. In addition, it -// returns list of remaining directories that don't exist yet. -func findExistingPrefix(base, pathname string) (string, []string, error) { - rel, err := filepath.Rel(base, pathname) - if err != nil { - return base, nil, err - } - - if startsWithBackstep(rel) { - return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base) - } - - if rel == "." { - // base and pathname are equal - return pathname, []string{}, nil - } - - dirs := strings.Split(rel, string(filepath.Separator)) - - parent := base - currentPath := base - for i, dir := range dirs { - parent = currentPath - currentPath = filepath.Join(parent, dir) - if _, err := os.Lstat(currentPath); err != nil { - if os.IsNotExist(err) { - return parent, dirs[i:], nil - } - return base, nil, err - } - } - - return pathname, []string{}, nil -} diff --git a/pkg/util/mount/mount_windows_test.go b/pkg/util/mount/mount_windows_test.go index e918f3f9efa..3ec233e095d 100644 --- a/pkg/util/mount/mount_windows_test.go +++ b/pkg/util/mount/mount_windows_test.go @@ -134,409 +134,6 @@ func TestGetMountRefs(t *testing.T) { } } -func TestDoSafeMakeDir(t *testing.T) { - base, err := ioutil.TempDir("", "TestDoSafeMakeDir") - if err != nil { - t.Fatalf(err.Error()) - } - - defer os.RemoveAll(base) - - testingVolumePath := filepath.Join(base, "testingVolumePath") - os.MkdirAll(testingVolumePath, 0755) - defer os.RemoveAll(testingVolumePath) - - tests := []struct { - volumePath string - subPath string - expectError bool - symlinkTarget string - }{ - { - volumePath: testingVolumePath, - subPath: ``, - expectError: true, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `x`), - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\c\d`), - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `symlink`), - expectError: false, - symlinkTarget: base, - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `symlink\c\d`), - expectError: true, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `symlink\y926`), - expectError: true, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\symlink`), - expectError: false, - symlinkTarget: base, - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\x\symlink`), - expectError: false, - symlinkTarget: filepath.Join(testingVolumePath, `a`), - }, - } - - for _, test := range tests { - if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 { - // make all parent sub directories - if parent := filepath.Dir(test.subPath); parent != "." { - os.MkdirAll(parent, 0755) - } - - // make last element as symlink - linkPath := test.subPath - if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) { - if err := makeLink(linkPath, test.symlinkTarget); err != nil { - t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err)) - } - } - } - - err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755)) - if test.expectError { - assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath) - continue - } - assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath) - if _, err := os.Stat(test.subPath); os.IsNotExist(err) { - t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath) - } - } -} - -func TestLockAndCheckSubPath(t *testing.T) { - base, err := ioutil.TempDir("", "TestLockAndCheckSubPath") - if err != nil { - t.Fatalf(err.Error()) - } - - defer os.RemoveAll(base) - - testingVolumePath := filepath.Join(base, "testingVolumePath") - - tests := []struct { - volumePath string - subPath string - expectedHandleCount int - expectError bool - symlinkTarget string - }{ - { - volumePath: `c:\`, - subPath: ``, - expectedHandleCount: 0, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: ``, - subPath: `a`, - expectedHandleCount: 0, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a`), - expectedHandleCount: 1, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\c\d`), - expectedHandleCount: 4, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `symlink`), - expectedHandleCount: 0, - expectError: true, - symlinkTarget: base, - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`), - expectedHandleCount: 0, - expectError: true, - symlinkTarget: base, - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`), - expectedHandleCount: 2, - expectError: false, - symlinkTarget: filepath.Join(testingVolumePath, `a\b`), - }, - } - - for _, test := range tests { - if len(test.volumePath) > 0 && len(test.subPath) > 0 { - os.MkdirAll(test.volumePath, 0755) - if len(test.symlinkTarget) == 0 { - // make all intermediate sub directories - os.MkdirAll(test.subPath, 0755) - } else { - // make all parent sub directories - if parent := filepath.Dir(test.subPath); parent != "." { - os.MkdirAll(parent, 0755) - } - - // make last element as symlink - linkPath := test.subPath - if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) { - if err := makeLink(linkPath, test.symlinkTarget); err != nil { - t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err)) - } - } - } - } - - fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath) - unlockPath(fileHandles) - assert.Equal(t, test.expectedHandleCount, len(fileHandles)) - if test.expectError { - assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) - continue - } - assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) - } - - // remove dir will happen after closing all file handles - assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath) -} - -func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) { - base, err := ioutil.TempDir("", "TestLockAndCheckSubPathWithoutSymlink") - if err != nil { - t.Fatalf(err.Error()) - } - - defer os.RemoveAll(base) - - testingVolumePath := filepath.Join(base, "testingVolumePath") - - tests := []struct { - volumePath string - subPath string - expectedHandleCount int - expectError bool - symlinkTarget string - }{ - { - volumePath: `c:\`, - subPath: ``, - expectedHandleCount: 0, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: ``, - subPath: `a`, - expectedHandleCount: 0, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a`), - expectedHandleCount: 1, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\c\d`), - expectedHandleCount: 4, - expectError: false, - symlinkTarget: "", - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `symlink`), - expectedHandleCount: 1, - expectError: true, - symlinkTarget: base, - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`), - expectedHandleCount: 4, - expectError: true, - symlinkTarget: base, - }, - { - volumePath: testingVolumePath, - subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`), - expectedHandleCount: 5, - expectError: true, - symlinkTarget: filepath.Join(testingVolumePath, `a\b`), - }, - } - - for _, test := range tests { - if len(test.volumePath) > 0 && len(test.subPath) > 0 { - os.MkdirAll(test.volumePath, 0755) - if len(test.symlinkTarget) == 0 { - // make all intermediate sub directories - os.MkdirAll(test.subPath, 0755) - } else { - // make all parent sub directories - if parent := filepath.Dir(test.subPath); parent != "." { - os.MkdirAll(parent, 0755) - } - - // make last element as symlink - linkPath := test.subPath - if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) { - if err := makeLink(linkPath, test.symlinkTarget); err != nil { - t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err)) - } - } - } - } - - fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath) - unlockPath(fileHandles) - assert.Equal(t, test.expectedHandleCount, len(fileHandles)) - if test.expectError { - assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) - continue - } - assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) - } - - // remove dir will happen after closing all file handles - assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath) -} - -func TestFindExistingPrefix(t *testing.T) { - base, err := ioutil.TempDir("", "TestFindExistingPrefix") - if err != nil { - t.Fatalf(err.Error()) - } - - defer os.RemoveAll(base) - - testingVolumePath := filepath.Join(base, "testingVolumePath") - - tests := []struct { - base string - pathname string - expectError bool - expectedExistingPath string - expectedToCreateDirs []string - createSubPathBeforeTest bool - }{ - { - base: `c:\tmp\a`, - pathname: `c:\tmp\b`, - expectError: true, - expectedExistingPath: "", - expectedToCreateDirs: []string{}, - createSubPathBeforeTest: false, - }, - { - base: ``, - pathname: `c:\tmp\b`, - expectError: true, - expectedExistingPath: "", - expectedToCreateDirs: []string{}, - createSubPathBeforeTest: false, - }, - { - base: `c:\tmp\a`, - pathname: `d:\tmp\b`, - expectError: true, - expectedExistingPath: "", - expectedToCreateDirs: []string{}, - createSubPathBeforeTest: false, - }, - { - base: testingVolumePath, - pathname: testingVolumePath, - expectError: false, - expectedExistingPath: testingVolumePath, - expectedToCreateDirs: []string{}, - createSubPathBeforeTest: false, - }, - { - base: testingVolumePath, - pathname: filepath.Join(testingVolumePath, `a\b`), - expectError: false, - expectedExistingPath: filepath.Join(testingVolumePath, `a\b`), - expectedToCreateDirs: []string{}, - createSubPathBeforeTest: true, - }, - { - base: testingVolumePath, - pathname: filepath.Join(testingVolumePath, `a\b\c\`), - expectError: false, - expectedExistingPath: filepath.Join(testingVolumePath, `a\b`), - expectedToCreateDirs: []string{`c`}, - createSubPathBeforeTest: false, - }, - { - base: testingVolumePath, - pathname: filepath.Join(testingVolumePath, `a\b\c\d`), - expectError: false, - expectedExistingPath: filepath.Join(testingVolumePath, `a\b`), - expectedToCreateDirs: []string{`c`, `d`}, - createSubPathBeforeTest: false, - }, - } - - for _, test := range tests { - if test.createSubPathBeforeTest { - os.MkdirAll(test.pathname, 0755) - } - - existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname) - if test.expectError { - assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname) - continue - } - assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname) - - assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q", - test.base, test.pathname, existingPath, test.expectedExistingPath) - - assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q", - test.base, test.pathname, toCreate, test.expectedToCreateDirs) - - } - // remove dir will happen after closing all file handles - assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath) -} - func TestPathWithinBase(t *testing.T) { tests := []struct { fullPath string diff --git a/pkg/util/mount/nsenter_mount.go b/pkg/util/mount/nsenter_mount.go index ce5d825da79..cdd39336de5 100644 --- a/pkg/util/mount/nsenter_mount.go +++ b/pkg/util/mount/nsenter_mount.go @@ -23,9 +23,7 @@ import ( "os" "path/filepath" "strings" - "syscall" - "golang.org/x/sys/unix" "k8s.io/klog" "k8s.io/utils/nsenter" utilpath "k8s.io/utils/path" @@ -298,51 +296,6 @@ func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error) return mounter.ne.EvalSymlinks(pathname, true) } -func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { - return doCleanSubPaths(mounter, podDir, volumeName) -} - -func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - // Bind-mount the subpath to avoid using symlinks in subpaths. - newHostPath, err = doNsEnterBindSubPath(mounter, subPath) - - // There is no action when the container starts. Bind-mount will be cleaned - // when container stops by CleanSubPaths. - cleanupAction = nil - return newHostPath, cleanupAction, err -} - -func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { - fullSubdirPath := filepath.Join(base, subdir) - evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */) - if err != nil { - return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err) - } - evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath) - - evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */) - if err != nil { - return fmt.Errorf("error resolving symlinks in %s: %s", base, err) - } - evaluatedBase = filepath.Clean(evaluatedBase) - - rootDir := filepath.Clean(mounter.rootDir) - if PathWithinBase(evaluatedBase, rootDir) { - // Base is in /var/lib/kubelet. This directory is shared between the - // container with kubelet and the host. We don't need to add '/rootfs'. - // This is useful when /rootfs is mounted as read-only - we can still - // create subpaths for paths in /var/lib/kubelet. - return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm) - } - - // Base is somewhere on the host's filesystem. Add /rootfs and try to make - // the directory there. - // This requires /rootfs to be writable. - kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath) - kubeletBase := mounter.ne.KubeletPath(evaluatedBase) - return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm) -} - func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) { exists, err := mounter.ExistsPath(pathname) if err != nil { @@ -358,95 +311,6 @@ func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) { return searchMountPoints(hostpath, hostProcMountinfoPath) } -func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) { - // Linux, kubelet runs in a container: - // - safely open the subpath - // - bind-mount the subpath to target (this can be unsafe) - // - check that we mounted the right thing by comparing device ID and inode - // of the subpath (via safely opened fd) and the target (that's under our - // control) - - // Evaluate all symlinks here once for all subsequent functions. - evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/) - if err != nil { - return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) - } - evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/) - if err != nil { - return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) - } - klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath) - subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath) - subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath) - - // Check the subpath is correct and open it - fd, err := safeOpenSubPath(mounter, subpath) - if err != nil { - return "", err - } - defer syscall.Close(fd) - - alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) - if err != nil { - return "", err - } - if alreadyMounted { - return bindPathTarget, nil - } - - success := false - defer func() { - // Cleanup subpath on error - if !success { - klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) - if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { - klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) - } - } - }() - - // Leap of faith: optimistically expect that nobody has modified previously - // expanded evalSubPath with evil symlinks and bind-mount it. - // Mount is done on the host! don't use kubelet path! - klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget) - if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { - return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err) - } - - // Check that the bind-mount target is the same inode and device as the - // source that we keept open, i.e. we mounted the right thing. - err = checkDeviceInode(fd, bindPathTarget) - if err != nil { - return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err) - } - - success = true - klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) - return bindPathTarget, nil -} - -// checkDeviceInode checks that opened file and path represent the same file. -func checkDeviceInode(fd int, path string) error { - var srcStat, dstStat unix.Stat_t - err := unix.Fstat(fd, &srcStat) - if err != nil { - return fmt.Errorf("error running fstat on subpath FD: %v", err) - } - - err = unix.Stat(path, &dstStat) - if err != nil { - return fmt.Errorf("error running fstat on %s: %v", path, err) - } - - if srcStat.Dev != dstStat.Dev { - return fmt.Errorf("different device number") - } - if srcStat.Ino != dstStat.Ino { - return fmt.Errorf("different inode") - } - return nil -} - func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) { hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */) if err != nil { diff --git a/pkg/util/mount/nsenter_mount_test.go b/pkg/util/mount/nsenter_mount_test.go index 1ed4bd222b3..84d04b164d5 100644 --- a/pkg/util/mount/nsenter_mount_test.go +++ b/pkg/util/mount/nsenter_mount_test.go @@ -23,10 +23,8 @@ import ( "os" "os/user" "path/filepath" - "strings" "testing" - "golang.org/x/sys/unix" "k8s.io/utils/nsenter" ) @@ -76,80 +74,6 @@ func TestParseFindMnt(t *testing.T) { } } -func TestCheckDeviceInode(t *testing.T) { - testDir, err := ioutil.TempDir("", "nsenter-mounter-device-") - if err != nil { - t.Fatalf("Cannot create temporary directory: %s", err) - } - defer os.RemoveAll(testDir) - - tests := []struct { - name string - srcPath string - dstPath string - expectError string - }{ - { - name: "the same file", - srcPath: filepath.Join(testDir, "1"), - dstPath: filepath.Join(testDir, "1"), - expectError: "", - }, - { - name: "different file on the same FS", - srcPath: filepath.Join(testDir, "2.1"), - dstPath: filepath.Join(testDir, "2.2"), - expectError: "different inode", - }, - { - name: "different file on different device", - srcPath: filepath.Join(testDir, "3"), - // /proc is always on a different "device" than /tmp (or $TEMP) - dstPath: "/proc/self/status", - expectError: "different device", - }, - } - - for _, test := range tests { - if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil { - t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err) - continue - } - - // Don't create dst if it exists - if _, err := os.Stat(test.dstPath); os.IsNotExist(err) { - if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil { - t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err) - continue - } - } else if err != nil { - t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err) - continue - } - - fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644) - if err != nil { - t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err) - continue - } - - err = checkDeviceInode(fd, test.dstPath) - - if test.expectError == "" && err != nil { - t.Errorf("Test %q: expected no error, got %s", test.name, err) - } - if test.expectError != "" { - if err == nil { - t.Errorf("Test %q: expected error, got none", test.name) - } else { - if !strings.Contains(err.Error(), test.expectError) { - t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err) - } - } - } - } -} - func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) { rootfsPath = filepath.Join(tmpdir, "rootfs") if err := os.Mkdir(rootfsPath, 0755); err != nil { @@ -504,221 +428,3 @@ func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) { } return fullPath, nil } - -func TestNsenterSafeMakeDir(t *testing.T) { - tests := []struct { - name string - prepare func(base, rootfs, varlib string) (expectedDir string, err error) - subdir string - expectError bool - // If true, "base" directory for SafeMakeDir will be /var/lib/kubelet - baseIsVarLib bool - }{ - { - name: "simple directory", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // expected to be created in /roots/ - expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure") - return expectedDir, nil - }, - }, - { - name: "simple existing directory", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: directory exists - hostPath := filepath.Join(base, "some/subdirectory/structure") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - // In rootfs: directory exists - kubeletPath := filepath.Join(rootfs, hostPath) - if err := os.MkdirAll(kubeletPath, 0755); err != nil { - return "", err - } - // expected to be created in /roots/ - expectedDir = kubeletPath - return expectedDir, nil - }, - }, - { - name: "absolute symlink into safe place", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /base/other/subdirectory exists, /base/some is link to /base/other - hostPath := filepath.Join(base, "other/subdirectory") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - somePath := filepath.Join(base, "some") - otherPath := filepath.Join(base, "other") - if err := os.Symlink(otherPath, somePath); err != nil { - return "", err - } - - // In rootfs: /base/other/subdirectory exists - kubeletPath := filepath.Join(rootfs, hostPath) - if err := os.MkdirAll(kubeletPath, 0755); err != nil { - return "", err - } - // expected 'structure' to be created - expectedDir = filepath.Join(rootfs, hostPath, "structure") - return expectedDir, nil - }, - }, - { - name: "relative symlink into safe place", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /base/other/subdirectory exists, /base/some is link to other - hostPath := filepath.Join(base, "other/subdirectory") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - somePath := filepath.Join(base, "some") - if err := os.Symlink("other", somePath); err != nil { - return "", err - } - - // In rootfs: /base/other/subdirectory exists - kubeletPath := filepath.Join(rootfs, hostPath) - if err := os.MkdirAll(kubeletPath, 0755); err != nil { - return "", err - } - // expected 'structure' to be created - expectedDir = filepath.Join(rootfs, hostPath, "structure") - return expectedDir, nil - }, - }, - { - name: "symlink into unsafe place", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /base/some is link to /bin/other - somePath := filepath.Join(base, "some") - if err := os.Symlink("/bin", somePath); err != nil { - return "", err - } - return "", nil - }, - expectError: true, - }, - { - name: "simple directory in /var/lib/kubelet", - // evaluated in varlib - subdir: "some/subdirectory/structure", - baseIsVarLib: true, - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // expected to be created in /base/var/lib/kubelet, not in /rootfs! - expectedDir = filepath.Join(varlib, "some/subdirectory/structure") - return expectedDir, nil - }, - }, - { - name: "safe symlink in /var/lib/kubelet", - // evaluated in varlib - subdir: "some/subdirectory/structure", - baseIsVarLib: true, - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other - hostPath := filepath.Join(varlib, "other/subdirectory") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - somePath := filepath.Join(varlib, "some") - if err := os.Symlink("other", somePath); err != nil { - return "", err - } - - // expected to be created in /base/var/lib/kubelet, not in /rootfs! - expectedDir = filepath.Join(varlib, "other/subdirectory/structure") - return expectedDir, nil - }, - }, - { - name: "unsafe symlink in /var/lib/kubelet", - // evaluated in varlib - subdir: "some/subdirectory/structure", - baseIsVarLib: true, - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /varlib/some is link to /bin - somePath := filepath.Join(varlib, "some") - if err := os.Symlink("/bin", somePath); err != nil { - return "", err - } - - return "", nil - }, - expectError: true, - }, - } - for _, test := range tests { - tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") - if err != nil { - t.Error(err) - continue - } - defer os.RemoveAll(tmpdir) - - mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t) - if err != nil { - t.Error(err) - continue - } - // Prepare base directory for the test - testBase := filepath.Join(tmpdir, "base") - if err := os.Mkdir(testBase, 0755); err != nil { - t.Error(err) - continue - } - // Prepare base directory also in /rootfs - rootfsBase := filepath.Join(rootfs, testBase) - if err := os.MkdirAll(rootfsBase, 0755); err != nil { - t.Error(err) - continue - } - - expectedDir := "" - if test.prepare != nil { - expectedDir, err = test.prepare(testBase, rootfs, varlib) - if err != nil { - t.Error(err) - continue - } - } - - if test.baseIsVarLib { - // use /var/lib/kubelet as the test base so we can test creating - // subdirs there directly in /var/lib/kubenet and not in - // /rootfs/var/lib/kubelet - testBase = varlib - } - - err = mounter.SafeMakeDir(test.subdir, testBase, 0755) - if err != nil && !test.expectError { - t.Errorf("Test %q: unexpected error: %s", test.name, err) - } - if test.expectError { - if err == nil { - t.Errorf("Test %q: expected error, got none", test.name) - } else { - if !strings.Contains(err.Error(), "is outside of allowed base") { - t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err) - } - } - } - - if expectedDir != "" { - _, err := os.Stat(expectedDir) - if err != nil { - t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err) - } - } - } -} diff --git a/pkg/util/mount/nsenter_mount_unsupported.go b/pkg/util/mount/nsenter_mount_unsupported.go index d68f751af84..4679b7e150c 100644 --- a/pkg/util/mount/nsenter_mount_unsupported.go +++ b/pkg/util/mount/nsenter_mount_unsupported.go @@ -93,18 +93,6 @@ func (*NsenterMounter) EvalHostSymlinks(pathname string) (string, error) { return "", errors.New("not implemented") } -func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { - return nil -} - -func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - return subPath.Path, nil, nil -} - -func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { - return nil -} - func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) { return nil, errors.New("not implemented") } diff --git a/pkg/volume/BUILD b/pkg/volume/BUILD index ba5d04ba078..2e39a0c3d02 100644 --- a/pkg/volume/BUILD +++ b/pkg/volume/BUILD @@ -21,6 +21,7 @@ go_library( "//pkg/util/mount:go_default_library", "//pkg/volume/util/fs:go_default_library", "//pkg/volume/util/recyclerclient:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 90b7efd9502..0bbd69640c2 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -36,6 +36,7 @@ import ( "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" + "k8s.io/kubernetes/pkg/volume/util/subpath" ) type ProbeOperation uint32 @@ -367,6 +368,9 @@ type VolumeHost interface { // Returns the event recorder of kubelet. GetEventRecorder() record.EventRecorder + + // Returns an interface that should be used to execute subpath operations + GetSubpather() subpath.Interface } // VolumePluginMgr tracks registered plugins. diff --git a/pkg/volume/testing/BUILD b/pkg/volume/testing/BUILD index f477a3996b4..02f4efba7c8 100644 --- a/pkg/volume/testing/BUILD +++ b/pkg/volume/testing/BUILD @@ -17,6 +17,7 @@ go_library( "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/recyclerclient:go_default_library", + "//pkg/volume/util/subpath:go_default_library", "//pkg/volume/util/volumepathhandler:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index b294f21bc44..d41ea0d4f96 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -42,6 +42,7 @@ import ( . "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" + "k8s.io/kubernetes/pkg/volume/util/subpath" "k8s.io/kubernetes/pkg/volume/util/volumepathhandler" utilstrings "k8s.io/utils/strings" ) @@ -71,6 +72,7 @@ type fakeVolumeHost struct { exec mount.Exec nodeLabels map[string]string nodeName string + subpather subpath.Interface } func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost { @@ -101,6 +103,7 @@ func newFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins [ } host.exec = mount.NewFakeExec(nil) host.pluginMgr.InitPlugins(plugins, nil /* prober */, host) + host.subpather = &subpath.FakeSubpath{} return host } @@ -149,6 +152,10 @@ func (f *fakeVolumeHost) GetMounter(pluginName string) mount.Interface { return f.mounter } +func (f *fakeVolumeHost) GetSubpather() subpath.Interface { + return f.subpather +} + func (f *fakeVolumeHost) NewWrapperMounter(volName string, spec Spec, pod *v1.Pod, opts VolumeOptions) (Mounter, error) { // The name of wrapper volume is set to "wrapped_{wrapped_volume_name}" wrapperVolumeName := "wrapped_" + volName diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD index dd48f977e19..0dec3818138 100644 --- a/pkg/volume/util/BUILD +++ b/pkg/volume/util/BUILD @@ -88,6 +88,7 @@ filegroup( "//pkg/volume/util/nestedpendingoperations:all-srcs", "//pkg/volume/util/operationexecutor:all-srcs", "//pkg/volume/util/recyclerclient:all-srcs", + "//pkg/volume/util/subpath:all-srcs", "//pkg/volume/util/types:all-srcs", "//pkg/volume/util/volumepathhandler:all-srcs", ], diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 9cbea4f3d70..66a183d98e1 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -750,11 +750,11 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc( } unmountVolumeFunc := func() (error, error) { - mounter := og.volumePluginMgr.Host.GetMounter(volumeToUnmount.PluginName) + subpather := og.volumePluginMgr.Host.GetSubpather() // Remove all bind-mounts for subPaths podDir := path.Join(podsDir, string(volumeToUnmount.PodUID)) - if err := mounter.CleanSubPaths(podDir, volumeToUnmount.InnerVolumeSpecName); err != nil { + if err := subpather.CleanSubPaths(podDir, volumeToUnmount.InnerVolumeSpecName); err != nil { return volumeToUnmount.GenerateError("error cleaning subPath mounts", err) } diff --git a/pkg/volume/util/subpath/BUILD b/pkg/volume/util/subpath/BUILD new file mode 100644 index 00000000000..ebbbd4d138e --- /dev/null +++ b/pkg/volume/util/subpath/BUILD @@ -0,0 +1,100 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "subpath.go", + "subpath_linux.go", + "subpath_nsenter.go", + "subpath_unsupported.go", + "subpath_windows.go", + ], + importpath = "k8s.io/kubernetes/pkg/volume/util/subpath", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:dragonfly": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:freebsd": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/util/mount:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:nacl": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:netbsd": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:openbsd": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:plan9": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:solaris": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//pkg/util/mount:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = [ + "subpath_linux_test.go", + "subpath_nsenter_test.go", + "subpath_windows_test.go", + ], + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/util/mount:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/nsenter:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/github.com/stretchr/testify/assert:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/volume/util/subpath/OWNERS b/pkg/volume/util/subpath/OWNERS new file mode 100644 index 00000000000..c8821ce7354 --- /dev/null +++ b/pkg/volume/util/subpath/OWNERS @@ -0,0 +1,13 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - jingxu97 + - saad-ali + - jsafrane + - msau42 + - andyzhangx +approvers: + - jingxu97 + - saad-ali + - jsafrane + diff --git a/pkg/volume/util/subpath/subpath.go b/pkg/volume/util/subpath/subpath.go new file mode 100644 index 00000000000..c843f6cf896 --- /dev/null +++ b/pkg/volume/util/subpath/subpath.go @@ -0,0 +1,92 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subpath + +import "os" + +// Interface defines the set of methods all subpathers must implement +type Interface interface { + // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given + // pod volume directory. + CleanSubPaths(poodDir string, volumeName string) error + + // PrepareSafeSubpath does everything that's necessary to prepare a subPath + // that's 1) inside given volumePath and 2) immutable after this call. + // + // newHostPath - location of prepared subPath. It should be used instead of + // hostName when running the container. + // cleanupAction - action to run when the container is running or it failed to start. + // + // CleanupAction must be called immediately after the container with given + // subpath starts. On the other hand, Interface.CleanSubPaths must be called + // when the pod finishes. + PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) + + // SafeMakeDir creates subdir within given base. It makes sure that the + // created directory does not escape given base directory mis-using + // symlinks. Note that the function makes sure that it creates the directory + // somewhere under the base, nothing else. E.g. if the directory already + // exists, it may exist outside of the base due to symlinks. + // This method should be used if the directory to create is inside volume + // that's under user control. User must not be able to use symlinks to + // escape the volume to create directories somewhere else. + SafeMakeDir(subdir string, base string, perm os.FileMode) error +} + +// Subpath defines the attributes of a subpath +type Subpath struct { + // index of the VolumeMount for this container + VolumeMountIndex int + + // Full path to the subpath directory on the host + Path string + + // name of the volume that is a valid directory name. + VolumeName string + + // Full path to the volume path + VolumePath string + + // Path to the pod's directory, including pod UID + PodDir string + + // Name of the container + ContainerName string +} + +// Compile time-check for all implementers of subpath interface +var _ Interface = &subpath{} +var _ Interface = &FakeSubpath{} + +// FakeSubpath is a subpather implementation for testing +type FakeSubpath struct{} + +// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns +// newHostPath == subPath.Path +func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, nil +} + +// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop +func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error { + return nil +} + +// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop +func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return nil +} diff --git a/pkg/volume/util/subpath/subpath_linux.go b/pkg/volume/util/subpath/subpath_linux.go index c66039e5e09..977f7d2da4d 100644 --- a/pkg/volume/util/subpath/subpath_linux.go +++ b/pkg/volume/util/subpath/subpath_linux.go @@ -16,44 +16,25 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( - "errors" "fmt" "io/ioutil" "os" - "os/exec" - "path" "path/filepath" "strconv" "strings" "syscall" "golang.org/x/sys/unix" - "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" - utilexec "k8s.io/utils/exec" - utilio "k8s.io/utils/io" - utilpath "k8s.io/utils/path" + + "k8s.io/kubernetes/pkg/util/mount" ) const ( - // How many times to retry for a consistent read of /proc/mounts. - maxListTries = 3 - // Number of fields per line in /proc/mounts as per the fstab man page. - expectedNumFieldsPerLine = 6 - // At least number of fields per line in /proc//mountinfo. - expectedAtLeastNumFieldsPerMountInfo = 10 - // Location of the mount file to use - procMountsPath = "/proc/mounts" - // Location of the mountinfo file - procMountInfoPath = "/proc/self/mountinfo" - // 'fsck' found errors and corrected them - fsckErrorsCorrected = 1 - // 'fsck' found errors but exited without correcting them - fsckErrorsUncorrected = 4 - // place for subpath mounts // TODO: pass in directory using kubelet_getters instead containerSubPathDirectoryName = "volume-subpaths" @@ -63,671 +44,34 @@ const ( openFDFlags = unix.O_NOFOLLOW | unix.O_PATH ) -// Mounter provides the default implementation of mount.Interface -// for the linux platform. This implementation assumes that the -// kubelet is running in the host's root mount namespace. -type Mounter struct { - mounterPath string - withSystemd bool +type subpath struct { + mounter mount.Interface } -// New returns a mount.Interface for the current system. -// It provides options to override the default mounter behavior. -// mounterPath allows using an alternative to `/bin/mount` for mounting. -func New(mounterPath string) Interface { - return &Mounter{ - mounterPath: mounterPath, - withSystemd: detectSystemd(), +// New returns a subpath.Interface for the current system +func New(mounter mount.Interface) Interface { + return &subpath{ + mounter: mounter, } } -// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must -// be an empty string in case it's not required, e.g. for remount, or for auto filesystem -// type, where kernel handles fstype for you. The mount 'options' is a list of options, -// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is -// required, call Mount with an empty string list or nil. -func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { - // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. - // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. - mounterPath := "" - bind, bindOpts, bindRemountOpts := isBind(options) - if bind { - err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts) - if err != nil { - return err - } - return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts) - } - // The list of filesystems that require containerized mounter on GCI image cluster - fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs") - if fsTypesNeedMounter.Has(fstype) { - mounterPath = mounter.mounterPath - } - return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options) +func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(sp.mounter, podDir, volumeName) } -// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. -func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error { - mountArgs := makeMountArgs(source, target, fstype, options) - if len(mounterPath) > 0 { - mountArgs = append([]string{mountCmd}, mountArgs...) - mountCmd = mounterPath - } - - if m.withSystemd { - // Try to run mount via systemd-run --scope. This will escape the - // service where kubelet runs and any fuse daemons will be started in a - // specific scope. kubelet service than can be restarted without killing - // these fuse daemons. - // - // Complete command line (when mounterPath is not used): - // systemd-run --description=... --scope -- mount -t - // - // Expected flow: - // * systemd-run creates a transient scope (=~ cgroup) and executes its - // argument (/bin/mount) there. - // * mount does its job, forks a fuse daemon if necessary and finishes. - // (systemd-run --scope finishes at this point, returning mount's exit - // code and stdout/stderr - thats one of --scope benefits). - // * systemd keeps the fuse daemon running in the scope (i.e. in its own - // cgroup) until the fuse daemon dies (another --scope benefit). - // Kubelet service can be restarted and the fuse daemon survives. - // * When the fuse daemon dies (e.g. during unmount) systemd removes the - // scope automatically. - // - // systemd-mount is not used because it's too new for older distros - // (CentOS 7, Debian Jessie). - mountCmd, mountArgs = addSystemdScope("systemd-run", target, mountCmd, mountArgs) - } else { - // No systemd-run on the host (or we failed to check it), assume kubelet - // does not run as a systemd service. - // No code here, mountCmd and mountArgs are already populated. - } - - klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) - command := exec.Command(mountCmd, mountArgs...) - output, err := command.CombinedOutput() +func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) if err != nil { - args := strings.Join(mountArgs, " ") - klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) - return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", - err, mountCmd, args, string(output)) + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) } - return err + + realFullPath := filepath.Join(realBase, subdir) + + return doSafeMakeDir(realFullPath, realBase, perm) } -// detectSystemd returns true if OS runs with systemd as init. When not sure -// (permission errors, ...), it returns false. -// There may be different ways how to detect systemd, this one makes sure that -// systemd-runs (needed by Mount()) works. -func detectSystemd() bool { - if _, err := exec.LookPath("systemd-run"); err != nil { - klog.V(2).Infof("Detected OS without systemd") - return false - } - // Try to run systemd-run --scope /bin/true, that should be enough - // to make sure that systemd is really running and not just installed, - // which happens when running in a container with a systemd-based image - // but with different pid 1. - cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") - output, err := cmd.CombinedOutput() - if err != nil { - klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") - klog.V(4).Infof("systemd-run failed with: %v", err) - klog.V(4).Infof("systemd-run output: %s", string(output)) - return false - } - klog.V(2).Infof("Detected OS with systemd") - return true -} - -// makeMountArgs makes the arguments to the mount(8) command. -func makeMountArgs(source, target, fstype string, options []string) []string { - // Build mount command as follows: - // mount [-t $fstype] [-o $options] [$source] $target - mountArgs := []string{} - if len(fstype) > 0 { - mountArgs = append(mountArgs, "-t", fstype) - } - if len(options) > 0 { - mountArgs = append(mountArgs, "-o", strings.Join(options, ",")) - } - if len(source) > 0 { - mountArgs = append(mountArgs, source) - } - mountArgs = append(mountArgs, target) - - return mountArgs -} - -// addSystemdScope adds "system-run --scope" to given command line -func addSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { - descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) - systemdRunArgs := []string{descriptionArg, "--scope", "--", command} - return systemdRunPath, append(systemdRunArgs, args...) -} - -// Unmount unmounts the target. -func (mounter *Mounter) Unmount(target string) error { - klog.V(4).Infof("Unmounting %s", target) - command := exec.Command("umount", target) - output, err := command.CombinedOutput() - if err != nil { - return fmt.Errorf("Unmount failed: %v\nUnmounting arguments: %s\nOutput: %s\n", err, target, string(output)) - } - return nil -} - -// List returns a list of all mounted filesystems. -func (*Mounter) List() ([]MountPoint, error) { - return listProcMounts(procMountsPath) -} - -func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { - deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) - return ((mp.Path == dir) || (mp.Path == deletedDir)) -} - -func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { - return isNotMountPoint(mounter, dir) -} - -// IsLikelyNotMountPoint determines if a directory is not a mountpoint. -// It is fast but not necessarily ALWAYS correct. If the path is in fact -// a bind mount from one part of a mount to another it will not be detected. -// mkdir /tmp/a /tmp/b; mount --bin /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") -// will return true. When in fact /tmp/b is a mount point. If this situation -// if of interest to you, don't use this function... -func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - stat, err := os.Stat(file) - if err != nil { - return true, err - } - rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/"))) - if err != nil { - return true, err - } - // If the directory has a different device as parent, then it is a mountpoint. - if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { - return false, nil - } - - return true, nil -} - -// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. -// If pathname is not a device, log and return false with nil error. -// If open returns errno EBUSY, return true with nil error. -// If open returns nil, return false with nil error. -// Otherwise, return false with error -func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { - return exclusiveOpenFailsOnDevice(pathname) -} - -// PathIsDevice uses FileInfo returned from os.Stat to check if path refers -// to a device. -func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { - pathType, err := mounter.GetFileType(pathname) - isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev - return isDevice, err -} - -func exclusiveOpenFailsOnDevice(pathname string) (bool, error) { - var isDevice bool - finfo, err := os.Stat(pathname) - if os.IsNotExist(err) { - isDevice = false - } - // err in call to os.Stat - if err != nil { - return false, fmt.Errorf( - "PathIsDevice failed for path %q: %v", - pathname, - err) - } - // path refers to a device - if finfo.Mode()&os.ModeDevice != 0 { - isDevice = true - } - - if !isDevice { - klog.Errorf("Path %q is not referring to a device.", pathname) - return false, nil - } - fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL, 0) - // If the device is in use, open will return an invalid fd. - // When this happens, it is expected that Close will fail and throw an error. - defer unix.Close(fd) - if errno == nil { - // device not in use - return false, nil - } else if errno == unix.EBUSY { - // device is in use - return true, nil - } - // error during call to Open - return false, errno -} - -//GetDeviceNameFromMount: given a mount point, find the device name from its global mount point -func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { - return getDeviceNameFromMount(mounter, mountPath, pluginDir) -} - -// getDeviceNameFromMount find the device name from /proc/mounts in which -// the mount path reference should match the given plugin directory. In case no mount path reference -// matches, returns the volume name taken from its given mountPath -func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { - refs, err := mounter.GetMountRefs(mountPath) - if err != nil { - klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) - return "", err - } - if len(refs) == 0 { - klog.V(4).Infof("Directory %s is not mounted", mountPath) - return "", fmt.Errorf("directory %s is not mounted", mountPath) - } - basemountPath := path.Join(pluginDir, MountsInGlobalPDPath) - for _, ref := range refs { - if strings.HasPrefix(ref, basemountPath) { - volumeID, err := filepath.Rel(basemountPath, ref) - if err != nil { - klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) - return "", err - } - return volumeID, nil - } - } - - return path.Base(mountPath), nil -} - -func listProcMounts(mountFilePath string) ([]MountPoint, error) { - content, err := utilio.ConsistentRead(mountFilePath, maxListTries) - if err != nil { - return nil, err - } - return parseProcMounts(content) -} - -func parseProcMounts(content []byte) ([]MountPoint, error) { - out := []MountPoint{} - lines := strings.Split(string(content), "\n") - for _, line := range lines { - if line == "" { - // the last split() item is empty string following the last \n - continue - } - fields := strings.Fields(line) - if len(fields) != expectedNumFieldsPerLine { - return nil, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line) - } - - mp := MountPoint{ - Device: fields[0], - Path: fields[1], - Type: fields[2], - Opts: strings.Split(fields[3], ","), - } - - freq, err := strconv.Atoi(fields[4]) - if err != nil { - return nil, err - } - mp.Freq = freq - - pass, err := strconv.Atoi(fields[5]) - if err != nil { - return nil, err - } - mp.Pass = pass - - out = append(out, mp) - } - return out, nil -} - -func (mounter *Mounter) MakeRShared(path string) error { - return doMakeRShared(path, procMountInfoPath) -} - -func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { - return getFileType(pathname) -} - -func (mounter *Mounter) MakeDir(pathname string) error { - err := os.MkdirAll(pathname, os.FileMode(0755)) - if err != nil { - if !os.IsExist(err) { - return err - } - } - return nil -} - -func (mounter *Mounter) MakeFile(pathname string) error { - f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) - defer f.Close() - if err != nil { - if !os.IsExist(err) { - return err - } - } - return nil -} - -func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { - return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) -} - -func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { - return filepath.EvalSymlinks(pathname) -} - -// formatAndMount uses unix utils to format and mount the given disk -func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { - readOnly := false - for _, option := range options { - if option == "ro" { - readOnly = true - break - } - } - - options = append(options, "defaults") - - if !readOnly { - // Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw. - klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) - args := []string{"-a", source} - out, err := mounter.Exec.Run("fsck", args...) - if err != nil { - ee, isExitError := err.(utilexec.ExitError) - switch { - case err == utilexec.ErrExecutableNotFound: - klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") - case isExitError && ee.ExitStatus() == fsckErrorsCorrected: - klog.Infof("Device %s has errors which were corrected by fsck.", source) - case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: - return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s.", source, string(out)) - case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: - klog.Infof("`fsck` error %s", string(out)) - } - } - } - - // Try to mount the disk - klog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target) - mountErr := mounter.Interface.Mount(source, target, fstype, options) - if mountErr != nil { - // Mount failed. This indicates either that the disk is unformatted or - // it contains an unexpected filesystem. - existingFormat, err := mounter.GetDiskFormat(source) - if err != nil { - return err - } - if existingFormat == "" { - if readOnly { - // Don't attempt to format if mounting as readonly, return an error to reflect this. - return errors.New("failed to mount unformatted volume as read only") - } - - // Disk is unformatted so format it. - args := []string{source} - // Use 'ext4' as the default - if len(fstype) == 0 { - fstype = "ext4" - } - - if fstype == "ext4" || fstype == "ext3" { - args = []string{ - "-F", // Force flag - "-m0", // Zero blocks reserved for super-user - source, - } - } - klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) - _, err := mounter.Exec.Run("mkfs."+fstype, args...) - if err == nil { - // the disk has been formatted successfully try to mount it again. - klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) - return mounter.Interface.Mount(source, target, fstype, options) - } - klog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err) - return err - } else { - // Disk is already formatted and failed to mount - if len(fstype) == 0 || fstype == existingFormat { - // This is mount error - return mountErr - } else { - // Block device is formatted with unexpected filesystem, let the user know - return fmt.Errorf("failed to mount the volume as %q, it already contains %s. Mount error: %v", fstype, existingFormat, mountErr) - } - } - } - return mountErr -} - -// GetDiskFormat uses 'blkid' to see if the given disk is unformated -func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { - args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} - klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) - dataOut, err := mounter.Exec.Run("blkid", args...) - output := string(dataOut) - klog.V(4).Infof("Output: %q, err: %v", output, err) - - if err != nil { - if exit, ok := err.(utilexec.ExitError); ok { - if exit.ExitStatus() == 2 { - // Disk device is unformatted. - // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was - // not found, or no (specified) devices could be identified, an - // exit code of 2 is returned. - return "", nil - } - } - klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) - return "", err - } - - var fstype, pttype string - - lines := strings.Split(output, "\n") - for _, l := range lines { - if len(l) <= 0 { - // Ignore empty line. - continue - } - cs := strings.Split(l, "=") - if len(cs) != 2 { - return "", fmt.Errorf("blkid returns invalid output: %s", output) - } - // TYPE is filesystem type, and PTTYPE is partition table type, according - // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. - if cs[0] == "TYPE" { - fstype = cs[1] - } else if cs[0] == "PTTYPE" { - pttype = cs[1] - } - } - - if len(pttype) > 0 { - klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) - // Returns a special non-empty string as filesystem type, then kubelet - // will not format it. - return "unknown data, probably partitions", nil - } - - return fstype, nil -} - -// isShared returns true, if given path is on a mount point that has shared -// mount propagation. -func isShared(mount string, mountInfoPath string) (bool, error) { - info, err := findMountInfo(mount, mountInfoPath) - if err != nil { - return false, err - } - - // parse optional parameters - for _, opt := range info.optionalFields { - if strings.HasPrefix(opt, "shared:") { - return true, nil - } - } - return false, nil -} - -// This represents a single line in /proc//mountinfo. -type mountInfo struct { - // Unique ID for the mount (maybe reused after umount). - id int - // The ID of the parent mount (or of self for the root of this mount namespace's mount tree). - parentID int - // The value of `st_dev` for files on this filesystem. - majorMinor string - // The pathname of the directory in the filesystem which forms the root of this mount. - root string - // Mount source, filesystem-specific information. e.g. device, tmpfs name. - source string - // Mount point, the pathname of the mount point. - mountPoint string - // Optional fieds, zero or more fields of the form "tag[:value]". - optionalFields []string - // The filesystem type in the form "type[.subtype]". - fsType string - // Per-mount options. - mountOptions []string - // Per-superblock options. - superOptions []string -} - -// parseMountInfo parses /proc/xxx/mountinfo. -func parseMountInfo(filename string) ([]mountInfo, error) { - content, err := utilio.ConsistentRead(filename, maxListTries) - if err != nil { - return []mountInfo{}, err - } - contentStr := string(content) - infos := []mountInfo{} - - for _, line := range strings.Split(contentStr, "\n") { - if line == "" { - // the last split() item is empty string following the last \n - continue - } - // See `man proc` for authoritative description of format of the file. - fields := strings.Fields(line) - if len(fields) < expectedAtLeastNumFieldsPerMountInfo { - return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line) - } - id, err := strconv.Atoi(fields[0]) - if err != nil { - return nil, err - } - parentID, err := strconv.Atoi(fields[1]) - if err != nil { - return nil, err - } - info := mountInfo{ - id: id, - parentID: parentID, - majorMinor: fields[2], - root: fields[3], - mountPoint: fields[4], - mountOptions: strings.Split(fields[5], ","), - } - // All fields until "-" are "optional fields". - i := 6 - for ; i < len(fields) && fields[i] != "-"; i++ { - info.optionalFields = append(info.optionalFields, fields[i]) - } - // Parse the rest 3 fields. - i += 1 - if len(fields)-i < 3 { - return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i) - } - info.fsType = fields[i] - info.source = fields[i+1] - info.superOptions = strings.Split(fields[i+2], ",") - infos = append(infos, info) - } - return infos, nil -} - -func findMountInfo(path, mountInfoPath string) (mountInfo, error) { - infos, err := parseMountInfo(mountInfoPath) - if err != nil { - return mountInfo{}, err - } - - // process /proc/xxx/mountinfo in backward order and find the first mount - // point that is prefix of 'path' - that's the mount where path resides - var info *mountInfo - for i := len(infos) - 1; i >= 0; i-- { - if PathWithinBase(path, infos[i].mountPoint) { - info = &infos[i] - break - } - } - if info == nil { - return mountInfo{}, fmt.Errorf("cannot find mount point for %q", path) - } - return *info, nil -} - -// doMakeRShared is common implementation of MakeRShared on Linux. It checks if -// path is shared and bind-mounts it as rshared if needed. mountCmd and -// mountArgs are expected to contain mount-like command, doMakeRShared will add -// '--bind ' and '--make-rshared ' to mountArgs. -func doMakeRShared(path string, mountInfoFilename string) error { - shared, err := isShared(path, mountInfoFilename) - if err != nil { - return err - } - if shared { - klog.V(4).Infof("Directory %s is already on a shared mount", path) - return nil - } - - klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path) - // mount --bind /var/lib/kubelet /var/lib/kubelet - if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil { - return fmt.Errorf("failed to bind-mount %s: %v", path, err) - } - - // mount --make-rshared /var/lib/kubelet - if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil { - return fmt.Errorf("failed to make %s rshared: %v", path, err) - } - - return nil -} - -// getSELinuxSupport is common implementation of GetSELinuxSupport on Linux. -func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) { - info, err := findMountInfo(path, mountInfoFilename) - if err != nil { - return false, err - } - - // "seclabel" can be both in mount options and super options. - for _, opt := range info.superOptions { - if opt == "seclabel" { - return true, nil - } - } - for _, opt := range info.mountOptions { - if opt == "seclabel" { - return true, nil - } - } - return false, nil -} - -func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - newHostPath, err = doBindSubPath(mounter, subPath) +func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + newHostPath, err = doBindSubPath(sp.mounter, subPath) // There is no action when the container starts. Bind-mount will be cleaned // when container stops by CleanSubPaths. @@ -735,9 +79,9 @@ func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, return newHostPath, cleanupAction, err } -// This implementation is shared between Linux and NsEnterMounter -func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) { - if !PathWithinBase(subpath.Path, subpath.VolumePath) { +// This implementation is shared between Linux and NsEnter +func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) { + if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) { return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath) } fd, err := doSafeOpen(subpath.Path, subpath.VolumePath) @@ -754,7 +98,7 @@ func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) { // This function is called also by NsEnterMounter. It works because // /var/lib/kubelet is mounted from the host into the container with Kubelet as // /var/lib/kubelet too. -func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) { +func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) { // Early check for already bind-mounted subpath. bindPathTarget := getSubpathBindTarget(subpath) notMount, err := mounter.IsNotMountPoint(bindPathTarget) @@ -805,7 +149,7 @@ func getSubpathBindTarget(subpath Subpath) string { return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex)) } -func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) { +func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) { // Linux, kubelet runs on the host: // - safely open the subpath // - bind-mount /proc//fd/ to subpath target @@ -864,12 +208,8 @@ func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err err return bindPathTarget, nil } -func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { - return doCleanSubPaths(mounter, podDir, volumeName) -} - -// This implementation is shared between Linux and NsEnterMounter -func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error { +// This implementation is shared between Linux and NsEnter +func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error { // scan /var/lib/kubelet/pods//volume-subpaths//* subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) @@ -929,12 +269,12 @@ func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error } // doCleanSubPath tears down the single subpath bind mount -func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error { +func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error { // process /var/lib/kubelet/pods//volume-subpaths/// klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) - if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil { + if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil { return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err) } @@ -943,7 +283,7 @@ func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string } // cleanSubPath will teardown the subpath bind mount and any remove any directories if empty -func cleanSubPath(mounter Interface, subpath Subpath) error { +func cleanSubPath(mounter mount.Interface, subpath Subpath) error { containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) // Clean subdir bindmount @@ -962,7 +302,7 @@ func cleanSubPath(mounter Interface, subpath Subpath) error { // removeEmptyDirs works backwards from endDir to baseDir and removes each directory // if it is empty. It stops once it encounters a directory that has content func removeEmptyDirs(baseDir, endDir string) error { - if !PathWithinBase(endDir, baseDir) { + if !mount.PathWithinBase(endDir, baseDir) { return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) } @@ -991,71 +331,13 @@ func removeEmptyDirs(baseDir, endDir string) error { return nil } -func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { - realBase, err := filepath.EvalSymlinks(base) - if err != nil { - return fmt.Errorf("error resolving symlinks in %s: %s", base, err) - } - - realFullPath := filepath.Join(realBase, subdir) - - return doSafeMakeDir(realFullPath, realBase, perm) -} - -func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { - if _, err := os.Stat(pathname); os.IsNotExist(err) { - return []string{}, nil - } else if err != nil { - return nil, err - } - realpath, err := filepath.EvalSymlinks(pathname) - if err != nil { - return nil, err - } - return searchMountPoints(realpath, procMountInfoPath) -} - -func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { - return getSELinuxSupport(pathname, procMountInfoPath) -} - -func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { - realpath, err := filepath.EvalSymlinks(pathname) - if err != nil { - return 0, err - } - return getFSGroup(realpath) -} - -func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { - return getMode(pathname) -} - -// This implementation is shared between Linux and NsEnterMounter -func getFSGroup(pathname string) (int64, error) { - info, err := os.Stat(pathname) - if err != nil { - return 0, err - } - return int64(info.Sys().(*syscall.Stat_t).Gid), nil -} - -// This implementation is shared between Linux and NsEnterMounter -func getMode(pathname string) (os.FileMode, error) { - info, err := os.Stat(pathname) - if err != nil { - return 0, err - } - return info.Mode(), nil -} - // This implementation is shared between Linux and NsEnterMounter. Both pathname // and base must be either already resolved symlinks or thet will be resolved in // kubelet's mount namespace (in case it runs containerized). func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { klog.V(4).Infof("Creating directory %q within base %q", pathname, base) - if !PathWithinBase(pathname, base) { + if !mount.PathWithinBase(pathname, base) { return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) } @@ -1082,7 +364,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { if err != nil { return fmt.Errorf("error opening directory %s: %s", existingPath, err) } - if !PathWithinBase(fullExistingPath, base) { + if !mount.PathWithinBase(fullExistingPath, base) { return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) } @@ -1244,7 +526,7 @@ func doSafeOpen(pathname string, base string) (int, error) { // sure the user cannot change already existing directories into symlinks. for _, seg := range segments { currentPath = filepath.Join(currentPath, seg) - if !PathWithinBase(currentPath, base) { + if !mount.PathWithinBase(currentPath, base) { return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) } @@ -1279,51 +561,3 @@ func doSafeOpen(pathname string, base string) (int, error) { return finalFD, nil } - -// searchMountPoints finds all mount references to the source, returns a list of -// mountpoints. -// This function assumes source cannot be device. -// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, -// it's possible to mount a non-root path of a filesystem, so we need to use -// root path and major:minor to represent mount source uniquely. -// This implementation is shared between Linux and NsEnterMounter -func searchMountPoints(hostSource, mountInfoPath string) ([]string, error) { - mis, err := parseMountInfo(mountInfoPath) - if err != nil { - return nil, err - } - - mountID := 0 - rootPath := "" - majorMinor := "" - - // Finding the underlying root path and major:minor if possible. - // We need search in backward order because it's possible for later mounts - // to overlap earlier mounts. - for i := len(mis) - 1; i >= 0; i-- { - if hostSource == mis[i].mountPoint || PathWithinBase(hostSource, mis[i].mountPoint) { - // If it's a mount point or path under a mount point. - mountID = mis[i].id - rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint)) - majorMinor = mis[i].majorMinor - break - } - } - - if rootPath == "" || majorMinor == "" { - return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) - } - - var refs []string - for i := range mis { - if mis[i].id == mountID { - // Ignore mount entry for mount source itself. - continue - } - if mis[i].root == rootPath && mis[i].majorMinor == majorMinor { - refs = append(refs, mis[i].mountPoint) - } - } - - return refs, nil -} diff --git a/pkg/volume/util/subpath/subpath_linux_test.go b/pkg/volume/util/subpath/subpath_linux_test.go index e2669dd0f7b..954a134157a 100644 --- a/pkg/volume/util/subpath/subpath_linux_test.go +++ b/pkg/volume/util/subpath/subpath_linux_test.go @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( "fmt" @@ -26,404 +26,14 @@ import ( "path/filepath" "reflect" "strconv" - "strings" "syscall" "testing" - "k8s.io/utils/exec" - "k8s.io/klog" + + "k8s.io/kubernetes/pkg/util/mount" ) -func TestReadProcMountsFrom(t *testing.T) { - successCase := - `/dev/0 /path/to/0 type0 flags 0 0 -/dev/1 /path/to/1 type1 flags 1 1 -/dev/2 /path/to/2 type2 flags,1,2=3 2 2 -` - // NOTE: readProcMountsFrom has been updated to using fnv.New32a() - mounts, err := parseProcMounts([]byte(successCase)) - if err != nil { - t.Errorf("expected success, got %v", err) - } - if len(mounts) != 3 { - t.Fatalf("expected 3 mounts, got %d", len(mounts)) - } - mp := MountPoint{"/dev/0", "/path/to/0", "type0", []string{"flags"}, 0, 0} - if !mountPointsEqual(&mounts[0], &mp) { - t.Errorf("got unexpected MountPoint[0]: %#v", mounts[0]) - } - mp = MountPoint{"/dev/1", "/path/to/1", "type1", []string{"flags"}, 1, 1} - if !mountPointsEqual(&mounts[1], &mp) { - t.Errorf("got unexpected MountPoint[1]: %#v", mounts[1]) - } - mp = MountPoint{"/dev/2", "/path/to/2", "type2", []string{"flags", "1", "2=3"}, 2, 2} - if !mountPointsEqual(&mounts[2], &mp) { - t.Errorf("got unexpected MountPoint[2]: %#v", mounts[2]) - } - - errorCases := []string{ - "/dev/0 /path/to/mount\n", - "/dev/1 /path/to/mount type flags a 0\n", - "/dev/2 /path/to/mount type flags 0 b\n", - } - for _, ec := range errorCases { - _, err := parseProcMounts([]byte(ec)) - if err == nil { - t.Errorf("expected error") - } - } -} - -func mountPointsEqual(a, b *MountPoint) bool { - if a.Device != b.Device || a.Path != b.Path || a.Type != b.Type || !reflect.DeepEqual(a.Opts, b.Opts) || a.Pass != b.Pass || a.Freq != b.Freq { - return false - } - return true -} - -func TestGetMountRefs(t *testing.T) { - fm := &FakeMounter{ - MountPoints: []MountPoint{ - {Device: "/dev/sdb", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd"}, - {Device: "/dev/sdb", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod"}, - {Device: "/dev/sdc", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2"}, - {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1"}, - {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2"}, - }, - } - - tests := []struct { - mountPath string - expectedRefs []string - }{ - { - "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod", - []string{ - "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd", - }, - }, - { - "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1", - []string{ - "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2", - "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2", - }, - }, - { - "/var/fake/directory/that/doesnt/exist", - []string{}, - }, - } - - for i, test := range tests { - if refs, err := fm.GetMountRefs(test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { - t.Errorf("%d. getMountRefs(%q) = %v, %v; expected %v, nil", i, test.mountPath, refs, err, test.expectedRefs) - } - } -} - -func setEquivalent(set1, set2 []string) bool { - map1 := make(map[string]bool) - map2 := make(map[string]bool) - for _, s := range set1 { - map1[s] = true - } - for _, s := range set2 { - map2[s] = true - } - - for s := range map1 { - if !map2[s] { - return false - } - } - for s := range map2 { - if !map1[s] { - return false - } - } - return true -} - -func TestGetDeviceNameFromMount(t *testing.T) { - fm := &FakeMounter{ - MountPoints: []MountPoint{ - {Device: "/dev/disk/by-path/prefix-lun-1", - Path: "/mnt/111"}, - {Device: "/dev/disk/by-path/prefix-lun-1", - Path: "/mnt/222"}, - }, - } - - tests := []struct { - mountPath string - expectedDevice string - expectedRefs int - }{ - { - "/mnt/222", - "/dev/disk/by-path/prefix-lun-1", - 2, - }, - } - - for i, test := range tests { - if device, refs, err := GetDeviceNameFromMount(fm, test.mountPath); err != nil || test.expectedRefs != refs || test.expectedDevice != device { - t.Errorf("%d. GetDeviceNameFromMount(%s) = (%s, %d), %v; expected (%s,%d), nil", i, test.mountPath, device, refs, err, test.expectedDevice, test.expectedRefs) - } - } -} - -func TestGetMountRefsByDev(t *testing.T) { - fm := &FakeMounter{ - MountPoints: []MountPoint{ - {Device: "/dev/sdb", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd"}, - {Device: "/dev/sdb", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod"}, - {Device: "/dev/sdc", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2"}, - {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1"}, - {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2"}, - }, - } - - tests := []struct { - mountPath string - expectedRefs []string - }{ - { - "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd", - []string{ - "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod", - }, - }, - { - "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2", - []string{ - "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1", - "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2", - }, - }, - } - - for i, test := range tests { - - if refs, err := getMountRefsByDev(fm, test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { - t.Errorf("%d. getMountRefsByDev(%q) = %v, %v; expected %v, nil", i, test.mountPath, refs, err, test.expectedRefs) - } - } -} - -func writeFile(content string) (string, string, error) { - tempDir, err := ioutil.TempDir("", "mounter_shared_test") - if err != nil { - return "", "", err - } - filename := filepath.Join(tempDir, "mountinfo") - err = ioutil.WriteFile(filename, []byte(content), 0600) - if err != nil { - os.RemoveAll(tempDir) - return "", "", err - } - return tempDir, filename, nil -} - -func TestIsSharedSuccess(t *testing.T) { - successMountInfo := - `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -76 62 8:1 / /boot rw,relatime shared:29 - ext4 /dev/sda1 rw,seclabel,data=ordered -78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel -80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw -82 62 0:43 / /var/lib/foo rw,relatime shared:32 - tmpfs tmpfs rw -83 63 0:44 / /var/lib/bar rw,relatime - tmpfs tmpfs rw -227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -` - tempDir, filename, err := writeFile(successMountInfo) - if err != nil { - t.Fatalf("cannot create temporary file: %v", err) - } - defer os.RemoveAll(tempDir) - - tests := []struct { - name string - path string - expectedResult bool - }{ - { - // /var/lib/kubelet is a directory on mount '/' that is shared - // This is the most common case. - "shared", - "/var/lib/kubelet", - true, - }, - { - // 8a2a... is a directory on mount /var/lib/docker/devicemapper - // that is private. - "private", - "/var/lib/docker/devicemapper/mnt/8a2a5c19eefb06d6f851dfcb240f8c113427f5b49b19658b5c60168e88267693/", - false, - }, - { - // 'directory' is a directory on mount - // /var/lib/docker/devicemapper/test/shared that is shared, but one - // of its parent is private. - "nested-shared", - "/var/lib/docker/devicemapper/test/shared/my/test/directory", - true, - }, - { - // /var/lib/foo is a mount point and it's shared - "shared-mount", - "/var/lib/foo", - true, - }, - { - // /var/lib/bar is a mount point and it's private - "private-mount", - "/var/lib/bar", - false, - }, - } - for _, test := range tests { - ret, err := isShared(test.path, filename) - if err != nil { - t.Errorf("test %s got unexpected error: %v", test.name, err) - } - if ret != test.expectedResult { - t.Errorf("test %s expected %v, got %v", test.name, test.expectedResult, ret) - } - } -} - -func TestIsSharedFailure(t *testing.T) { - errorTests := []struct { - name string - content string - }{ - { - // the first line is too short - name: "too-short-line", - content: `62 0 253:0 / / rw,relatime -76 62 8:1 / /boot rw,relatime shared:29 - ext4 /dev/sda1 rw,seclabel,data=ordered -78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel -80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw -227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -`, - }, - { - // there is no root mount - name: "no-root-mount", - content: `76 62 8:1 / /boot rw,relatime shared:29 - ext4 /dev/sda1 rw,seclabel,data=ordered -78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel -80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw -227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -`, - }, - } - for _, test := range errorTests { - tempDir, filename, err := writeFile(test.content) - if err != nil { - t.Fatalf("cannot create temporary file: %v", err) - } - defer os.RemoveAll(tempDir) - - _, err = isShared("/", filename) - if err == nil { - t.Errorf("test %q: expected error, got none", test.name) - } - } -} - -func TestPathWithinBase(t *testing.T) { - tests := []struct { - name string - fullPath string - basePath string - expected bool - }{ - { - name: "good subpath", - fullPath: "/a/b/c", - basePath: "/a", - expected: true, - }, - { - name: "good subpath 2", - fullPath: "/a/b/c", - basePath: "/a/b", - expected: true, - }, - { - name: "good subpath end slash", - fullPath: "/a/b/c/", - basePath: "/a/b", - expected: true, - }, - { - name: "good subpath backticks", - fullPath: "/a/b/../c", - basePath: "/a", - expected: true, - }, - { - name: "good subpath equal", - fullPath: "/a/b/c", - basePath: "/a/b/c", - expected: true, - }, - { - name: "good subpath equal 2", - fullPath: "/a/b/c/", - basePath: "/a/b/c", - expected: true, - }, - { - name: "good subpath root", - fullPath: "/a", - basePath: "/", - expected: true, - }, - { - name: "bad subpath parent", - fullPath: "/a/b/c", - basePath: "/a/b/c/d", - expected: false, - }, - { - name: "bad subpath outside", - fullPath: "/b/c", - basePath: "/a/b/c", - expected: false, - }, - { - name: "bad subpath prefix", - fullPath: "/a/b/cd", - basePath: "/a/b/c", - expected: false, - }, - { - name: "bad subpath backticks", - fullPath: "/a/../b", - basePath: "/a", - expected: false, - }, - { - name: "configmap subpath", - fullPath: "/var/lib/kubelet/pods/uuid/volumes/kubernetes.io~configmap/config/..timestamp/file.txt", - basePath: "/var/lib/kubelet/pods/uuid/volumes/kubernetes.io~configmap/config", - expected: true, - }, - } - for _, test := range tests { - if PathWithinBase(test.fullPath, test.basePath) != test.expected { - t.Errorf("test %q failed: expected %v", test.name, test.expected) - } - - } -} - func TestSafeMakeDir(t *testing.T) { defaultPerm := os.FileMode(0750) + os.ModeDir tests := []struct { @@ -795,14 +405,14 @@ func TestCleanSubPaths(t *testing.T) { name string // Function that prepares directory structure for the test under given // base. - prepare func(base string) ([]MountPoint, error) + prepare func(base string) ([]mount.MountPoint, error) // Function that validates directory structure after the test validate func(base string) error expectError bool }{ { name: "not-exists", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { return nil, nil }, validate: func(base string) error { @@ -812,7 +422,7 @@ func TestCleanSubPaths(t *testing.T) { }, { name: "subpath-not-mount", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { return nil, os.MkdirAll(filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0"), defaultPerm) }, validate: func(base string) error { @@ -822,7 +432,7 @@ func TestCleanSubPaths(t *testing.T) { }, { name: "subpath-file", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1") if err := os.MkdirAll(path, defaultPerm); err != nil { return nil, err @@ -836,7 +446,7 @@ func TestCleanSubPaths(t *testing.T) { }, { name: "subpath-container-not-dir", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { path := filepath.Join(base, containerSubPathDirectoryName, testVol) if err := os.MkdirAll(path, defaultPerm); err != nil { return nil, err @@ -850,7 +460,7 @@ func TestCleanSubPaths(t *testing.T) { }, { name: "subpath-multiple-container-not-dir", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { path := filepath.Join(base, containerSubPathDirectoryName, testVol) if err := os.MkdirAll(filepath.Join(path, "container1"), defaultPerm); err != nil { return nil, err @@ -868,12 +478,12 @@ func TestCleanSubPaths(t *testing.T) { }, { name: "subpath-mount", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") if err := os.MkdirAll(path, defaultPerm); err != nil { return nil, err } - mounts := []MountPoint{{Device: "/dev/sdb", Path: path}} + mounts := []mount.MountPoint{{Device: "/dev/sdb", Path: path}} return mounts, nil }, validate: func(base string) error { @@ -882,7 +492,7 @@ func TestCleanSubPaths(t *testing.T) { }, { name: "subpath-mount-multiple", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") path2 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "1") path3 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container2", "1") @@ -895,7 +505,7 @@ func TestCleanSubPaths(t *testing.T) { if err := os.MkdirAll(path3, defaultPerm); err != nil { return nil, err } - mounts := []MountPoint{ + mounts := []mount.MountPoint{ {Device: "/dev/sdb", Path: path}, {Device: "/dev/sdb", Path: path3}, } @@ -907,7 +517,7 @@ func TestCleanSubPaths(t *testing.T) { }, { name: "subpath-mount-multiple-vols", - prepare: func(base string) ([]MountPoint, error) { + prepare: func(base string) ([]mount.MountPoint, error) { path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") path2 := filepath.Join(base, containerSubPathDirectoryName, "vol2", "container1", "1") if err := os.MkdirAll(path, defaultPerm); err != nil { @@ -916,7 +526,7 @@ func TestCleanSubPaths(t *testing.T) { if err := os.MkdirAll(path2, defaultPerm); err != nil { return nil, err } - mounts := []MountPoint{ + mounts := []mount.MountPoint{ {Device: "/dev/sdb", Path: path}, } return mounts, nil @@ -943,7 +553,7 @@ func TestCleanSubPaths(t *testing.T) { t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) } - fm := &FakeMounter{MountPoints: mounts} + fm := &mount.FakeMounter{MountPoints: mounts} err = doCleanSubPaths(fm, base, testVol) if err != nil && !test.expectError { @@ -967,12 +577,12 @@ var ( testSubpath = 1 ) -func setupFakeMounter(testMounts []string) *FakeMounter { - mounts := []MountPoint{} +func setupFakeMounter(testMounts []string) *mount.FakeMounter { + mounts := []mount.MountPoint{} for _, mountPoint := range testMounts { - mounts = append(mounts, MountPoint{Device: "/foo", Path: mountPoint}) + mounts = append(mounts, mount.MountPoint{Device: "/foo", Path: mountPoint}) } - return &FakeMounter{MountPoints: mounts} + return &mount.FakeMounter{MountPoints: mounts} } func getTestPaths(base string) (string, string) { @@ -1240,227 +850,6 @@ func TestBindSubPath(t *testing.T) { } } -func TestParseMountInfo(t *testing.T) { - info := - `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel -80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw -82 62 0:43 / /var/lib/foo rw,relatime shared:32 - tmpfs tmpfs rw -83 63 0:44 / /var/lib/bar rw,relatime - tmpfs tmpfs rw -227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -76 17 8:1 / /mnt/stateful_partition rw,nosuid,nodev,noexec,relatime - ext4 /dev/sda1 rw,commit=30,data=ordered -80 17 8:1 /var /var rw,nosuid,nodev,noexec,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered -189 80 8:1 /var/lib/kubelet /var/lib/kubelet rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered -818 77 8:40 / /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered -819 78 8:48 / /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered -900 100 8:48 /dir1 /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volume-subpaths/vol1/subpath1/0 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered -901 101 8:1 /dir1 /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volume-subpaths/vol1/subpath1/1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered -902 102 8:1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volumes/kubernetes.io~empty-dir/vol1/dir1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volume-subpaths/vol1/subpath1/0 rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered -903 103 8:1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volumes/kubernetes.io~empty-dir/vol2/dir1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volume-subpaths/vol1/subpath1/1 rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered -178 25 253:0 /etc/bar /var/lib/kubelet/pods/12345/volume-subpaths/vol1/subpath1/0 rw,relatime shared:1 - ext4 /dev/sdb2 rw,errors=remount-ro,data=ordered -698 186 0:41 /tmp1/dir1 /var/lib/kubelet/pods/41135147-e697-11e7-9342-42010a800002/volume-subpaths/vol1/subpath1/0 rw shared:26 - tmpfs tmpfs rw -918 77 8:50 / /var/lib/kubelet/pods/2345/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered -919 78 8:58 / /var/lib/kubelet/pods/2345/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered -920 100 8:50 /dir1 /var/lib/kubelet/pods/2345/volume-subpaths/vol1/subpath1/0 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered -150 23 1:58 / /media/nfs_vol rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 -151 24 1:58 / /media/nfs_bindmount rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs/foo rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 -134 23 0:58 / /var/lib/kubelet/pods/43219158-e5e1-11e7-a392-0e858b8eaf40/volumes/kubernetes.io~nfs/nfs1 rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 -187 23 0:58 / /var/lib/kubelet/pods/1fc5ea21-eff4-11e7-ac80-0e858b8eaf40/volumes/kubernetes.io~nfs/nfs2 rw,relatime shared:96 - nfs4 172.18.4.223:/srv/nfs2 rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 -188 24 0:58 / /var/lib/kubelet/pods/43219158-e5e1-11e7-a392-0e858b8eaf40/volume-subpaths/nfs1/subpath1/0 rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs/foo rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 -347 60 0:71 / /var/lib/kubelet/pods/13195d46-f9fa-11e7-bbf1-5254007a695a/volumes/kubernetes.io~nfs/vol2 rw,relatime shared:170 - nfs 172.17.0.3:/exports/2 rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.17.0.3,mountvers=3,mountport=20048,mountproto=udp,local_lock=none,addr=172.17.0.3 -222 24 253:0 /tmp/src /mnt/dst rw,relatime shared:1 - ext4 /dev/mapper/vagrant--vg-root rw,errors=remount-ro,data=ordered -28 18 0:24 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:9 - tmpfs tmpfs ro,mode=755 -29 28 0:25 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd -31 28 0:27 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:13 - cgroup cgroup rw,cpuset -32 28 0:28 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:14 - cgroup cgroup rw,cpu,cpuacct -33 28 0:29 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime shared:15 - cgroup cgroup rw,freezer -34 28 0:30 / /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime shared:16 - cgroup cgroup rw,net_cls,net_prio -35 28 0:31 / /sys/fs/cgroup/pids rw,nosuid,nodev,noexec,relatime shared:17 - cgroup cgroup rw,pids -36 28 0:32 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:18 - cgroup cgroup rw,devices -37 28 0:33 / /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime shared:19 - cgroup cgroup rw,hugetlb -38 28 0:34 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime shared:20 - cgroup cgroup rw,blkio -39 28 0:35 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:21 - cgroup cgroup rw,memory -40 28 0:36 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime shared:22 - cgroup cgroup rw,perf_event -` - tempDir, filename, err := writeFile(info) - if err != nil { - t.Fatalf("cannot create temporary file: %v", err) - } - defer os.RemoveAll(tempDir) - - tests := []struct { - name string - id int - expectedInfo mountInfo - }{ - { - "simple bind mount", - 189, - mountInfo{ - id: 189, - parentID: 80, - majorMinor: "8:1", - root: "/var/lib/kubelet", - source: "/dev/sda1", - mountPoint: "/var/lib/kubelet", - optionalFields: []string{"shared:30"}, - fsType: "ext4", - mountOptions: []string{"rw", "relatime"}, - superOptions: []string{"rw", "commit=30", "data=ordered"}, - }, - }, - { - "bind mount a directory", - 222, - mountInfo{ - id: 222, - parentID: 24, - majorMinor: "253:0", - root: "/tmp/src", - source: "/dev/mapper/vagrant--vg-root", - mountPoint: "/mnt/dst", - optionalFields: []string{"shared:1"}, - fsType: "ext4", - mountOptions: []string{"rw", "relatime"}, - superOptions: []string{"rw", "errors=remount-ro", "data=ordered"}, - }, - }, - { - "more than one optional fields", - 224, - mountInfo{ - id: 224, - parentID: 62, - majorMinor: "253:0", - root: "/var/lib/docker/devicemapper/test/shared", - source: "/dev/mapper/ssd-root", - mountPoint: "/var/lib/docker/devicemapper/test/shared", - optionalFields: []string{"master:1", "shared:44"}, - fsType: "ext4", - mountOptions: []string{"rw", "relatime"}, - superOptions: []string{"rw", "seclabel", "data=ordered"}, - }, - }, - { - "cgroup-mountpoint", - 28, - mountInfo{ - id: 28, - parentID: 18, - majorMinor: "0:24", - root: "/", - source: "tmpfs", - mountPoint: "/sys/fs/cgroup", - optionalFields: []string{"shared:9"}, - fsType: "tmpfs", - mountOptions: []string{"ro", "nosuid", "nodev", "noexec"}, - superOptions: []string{"ro", "mode=755"}, - }, - }, - { - "cgroup-subsystem-systemd-mountpoint", - 29, - mountInfo{ - id: 29, - parentID: 28, - majorMinor: "0:25", - root: "/", - source: "cgroup", - mountPoint: "/sys/fs/cgroup/systemd", - optionalFields: []string{"shared:10"}, - fsType: "cgroup", - mountOptions: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}, - superOptions: []string{"rw", "xattr", "release_agent=/lib/systemd/systemd-cgroups-agent", "name=systemd"}, - }, - }, - { - "cgroup-subsystem-cpuset-mountpoint", - 31, - mountInfo{ - id: 31, - parentID: 28, - majorMinor: "0:27", - root: "/", - source: "cgroup", - mountPoint: "/sys/fs/cgroup/cpuset", - optionalFields: []string{"shared:13"}, - fsType: "cgroup", - mountOptions: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}, - superOptions: []string{"rw", "cpuset"}, - }, - }, - } - - infos, err := parseMountInfo(filename) - if err != nil { - t.Fatalf("Cannot parse %s: %s", filename, err) - } - - for _, test := range tests { - found := false - for _, info := range infos { - if info.id == test.id { - found = true - if !reflect.DeepEqual(info, test.expectedInfo) { - t.Errorf("Test case %q:\n expected: %+v\n got: %+v", test.name, test.expectedInfo, info) - } - break - } - } - if !found { - t.Errorf("Test case %q: mountPoint %d not found", test.name, test.id) - } - } -} - -func TestGetSELinuxSupport(t *testing.T) { - info := - `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel -83 63 0:44 / /var/lib/bar rw,relatime - tmpfs tmpfs rw -227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered -150 23 1:58 / /media/nfs_vol rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 -` - tempDir, filename, err := writeFile(info) - if err != nil { - t.Fatalf("cannot create temporary file: %v", err) - } - defer os.RemoveAll(tempDir) - - tests := []struct { - name string - mountPoint string - expectedResult bool - }{ - { - "ext4 on /", - "/", - true, - }, - { - "tmpfs on /var/lib/bar", - "/var/lib/bar", - false, - }, - { - "nfsv4", - "/media/nfs_vol", - false, - }, - } - - for _, test := range tests { - out, err := getSELinuxSupport(test.mountPoint, filename) - if err != nil { - t.Errorf("Test %s failed with error: %s", test.name, err) - } - if test.expectedResult != out { - t.Errorf("Test %s failed: expected %v, got %v", test.name, test.expectedResult, out) - } - } -} - func TestSafeOpen(t *testing.T) { defaultPerm := os.FileMode(0750) @@ -1796,259 +1185,40 @@ func TestFindExistingPrefix(t *testing.T) { } } -func TestGetFileType(t *testing.T) { - mounter := Mounter{"fake/path", false} - - testCase := []struct { - name string - expectedType FileType - setUp func() (string, string, error) - }{ - { - "Directory Test", - FileTypeDirectory, - func() (string, string, error) { - tempDir, err := ioutil.TempDir("", "test-get-filetype-") - return tempDir, tempDir, err - }, - }, - { - "File Test", - FileTypeFile, - func() (string, string, error) { - tempFile, err := ioutil.TempFile("", "test-get-filetype") - if err != nil { - return "", "", err - } - tempFile.Close() - return tempFile.Name(), tempFile.Name(), nil - }, - }, - { - "Socket Test", - FileTypeSocket, - func() (string, string, error) { - tempDir, err := ioutil.TempDir("", "test-get-filetype-") - if err != nil { - return "", "", err - } - tempSocketFile, err := createSocketFile(tempDir) - return tempSocketFile, tempDir, err - }, - }, - { - "Block Device Test", - FileTypeBlockDev, - func() (string, string, error) { - tempDir, err := ioutil.TempDir("", "test-get-filetype-") - if err != nil { - return "", "", err - } - - tempBlockFile := filepath.Join(tempDir, "test_blk_dev") - outputBytes, err := exec.New().Command("mknod", tempBlockFile, "b", "89", "1").CombinedOutput() - if err != nil { - err = fmt.Errorf("%v: %s ", err, outputBytes) - } - return tempBlockFile, tempDir, err - }, - }, - { - "Character Device Test", - FileTypeCharDev, - func() (string, string, error) { - tempDir, err := ioutil.TempDir("", "test-get-filetype-") - if err != nil { - return "", "", err - } - - tempCharFile := filepath.Join(tempDir, "test_char_dev") - outputBytes, err := exec.New().Command("mknod", tempCharFile, "c", "89", "1").CombinedOutput() - if err != nil { - err = fmt.Errorf("%v: %s ", err, outputBytes) - } - return tempCharFile, tempDir, err - }, - }, - } - - for idx, tc := range testCase { - path, cleanUpPath, err := tc.setUp() - if err != nil { - // Locally passed, but upstream CI is not friendly to create such device files - // Leave "Operation not permitted" out, which can be covered in an e2e test - if isOperationNotPermittedError(err) { - continue - } - t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) - } - if len(cleanUpPath) > 0 { - defer os.RemoveAll(cleanUpPath) - } - - fileType, err := mounter.GetFileType(path) - if err != nil { - t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) - } - if fileType != tc.expectedType { - t.Fatalf("[%d-%s] expected %s, but got %s", idx, tc.name, tc.expectedType, fileType) - } - } -} - -func isOperationNotPermittedError(err error) bool { - if strings.Contains(err.Error(), "Operation not permitted") { - return true - } - return false -} - -func TestSearchMountPoints(t *testing.T) { - base := ` -19 25 0:18 / /sys rw,nosuid,nodev,noexec,relatime shared:7 - sysfs sysfs rw -20 25 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:12 - proc proc rw -21 25 0:6 / /dev rw,nosuid,relatime shared:2 - devtmpfs udev rw,size=4058156k,nr_inodes=1014539,mode=755 -22 21 0:14 / /dev/pts rw,nosuid,noexec,relatime shared:3 - devpts devpts rw,gid=5,mode=620,ptmxmode=000 -23 25 0:19 / /run rw,nosuid,noexec,relatime shared:5 - tmpfs tmpfs rw,size=815692k,mode=755 -25 0 252:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered -26 19 0:12 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:8 - securityfs securityfs rw -27 21 0:21 / /dev/shm rw,nosuid,nodev shared:4 - tmpfs tmpfs rw -28 23 0:22 / /run/lock rw,nosuid,nodev,noexec,relatime shared:6 - tmpfs tmpfs rw,size=5120k -29 19 0:23 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:9 - tmpfs tmpfs ro,mode=755 -30 29 0:24 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd -31 19 0:25 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:11 - pstore pstore rw -32 29 0:26 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:13 - cgroup cgroup rw,devices -33 29 0:27 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime shared:14 - cgroup cgroup rw,freezer -34 29 0:28 / /sys/fs/cgroup/pids rw,nosuid,nodev,noexec,relatime shared:15 - cgroup cgroup rw,pids -35 29 0:29 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime shared:16 - cgroup cgroup rw,blkio -36 29 0:30 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:17 - cgroup cgroup rw,memory -37 29 0:31 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime shared:18 - cgroup cgroup rw,perf_event -38 29 0:32 / /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime shared:19 - cgroup cgroup rw,hugetlb -39 29 0:33 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:20 - cgroup cgroup rw,cpu,cpuacct -40 29 0:34 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:21 - cgroup cgroup rw,cpuset -41 29 0:35 / /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime shared:22 - cgroup cgroup rw,net_cls,net_prio -58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordere -` - - testcases := []struct { - name string - source string - mountInfos string - expectedRefs []string - expectedErr error - }{ - { - "dir", - "/mnt/disks/vol1", - base, - nil, - nil, - }, - { - "dir-used", - "/mnt/disks/vol1", - base + ` -56 25 252:0 /mnt/disks/vol1 /var/lib/kubelet/pods/1890aef5-5a60-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered -57 25 0:45 / /mnt/disks/vol rw,relatime shared:36 - tmpfs tmpfs rw -`, - []string{"/var/lib/kubelet/pods/1890aef5-5a60-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, - nil, - }, - { - "tmpfs-vol", - "/mnt/disks/vol1", - base + `120 25 0:76 / /mnt/disks/vol1 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k -`, - nil, - nil, - }, - { - "tmpfs-vol-used-by-two-pods", - "/mnt/disks/vol1", - base + `120 25 0:76 / /mnt/disks/vol1 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k -196 25 0:76 / /var/lib/kubelet/pods/ade3ac21-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k -228 25 0:76 / /var/lib/kubelet/pods/ac60532d-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k -`, - []string{ - "/var/lib/kubelet/pods/ade3ac21-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585", - "/var/lib/kubelet/pods/ac60532d-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585", - }, - nil, - }, - { - "tmpfs-subdir-used-indirectly-via-bindmount-dir-by-one-pod", - "/mnt/vol1/foo", - base + `177 25 0:46 / /mnt/data rw,relatime shared:37 - tmpfs data rw -190 25 0:46 /vol1 /mnt/vol1 rw,relatime shared:37 - tmpfs data rw -191 25 0:46 /vol2 /mnt/vol2 rw,relatime shared:37 - tmpfs data rw -62 25 0:46 /vol1/foo /var/lib/kubelet/pods/e25f2f01-5b06-11e8-8694-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:37 - tmpfs data rw -`, - []string{"/var/lib/kubelet/pods/e25f2f01-5b06-11e8-8694-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, - nil, - }, - { - "dir-bindmounted", - "/mnt/disks/vol2", - base + `342 25 252:0 /mnt/disks/vol2 /mnt/disks/vol2 rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered -`, - nil, - nil, - }, - { - "dir-bindmounted-used-by-one-pod", - "/mnt/disks/vol2", - base + `342 25 252:0 /mnt/disks/vol2 /mnt/disks/vol2 rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered -77 25 252:0 /mnt/disks/vol2 /var/lib/kubelet/pods/f30dc360-5a5d-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-1fb30a1c rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered -`, - []string{"/var/lib/kubelet/pods/f30dc360-5a5d-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-1fb30a1c"}, - nil, - }, - { - "blockfs", - "/mnt/disks/blkvol1", - base + `58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered -`, - nil, - nil, - }, - { - "blockfs-used-by-one-pod", - "/mnt/disks/blkvol1", - base + `58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered -62 25 7:1 / /var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered -`, - []string{"/var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, - nil, - }, - { - "blockfs-used-by-two-pods", - "/mnt/disks/blkvol1", - base + `58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered -62 25 7:1 / /var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered -95 25 7:1 / /var/lib/kubelet/pods/4854a48b-5a64-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered -`, - []string{"/var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test", - "/var/lib/kubelet/pods/4854a48b-5a64-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, - nil, - }, - } - tmpFile, err := ioutil.TempFile("", "test-get-filetype") +func validateDirEmpty(dir string) error { + files, err := ioutil.ReadDir(dir) if err != nil { - t.Fatal(err) + return err } - defer os.Remove(tmpFile.Name()) - defer tmpFile.Close() - for _, v := range testcases { - tmpFile.Truncate(0) - tmpFile.Seek(0, 0) - tmpFile.WriteString(v.mountInfos) - tmpFile.Sync() - refs, err := searchMountPoints(v.source, tmpFile.Name()) - if !reflect.DeepEqual(refs, v.expectedRefs) { - t.Errorf("test %q: expected Refs: %#v, got %#v", v.name, v.expectedRefs, refs) - } - if !reflect.DeepEqual(err, v.expectedErr) { - t.Errorf("test %q: expected err: %v, got %v", v.name, v.expectedErr, err) - } + + if len(files) != 0 { + return fmt.Errorf("Directory %q is not empty", dir) } + return nil +} + +func validateDirExists(dir string) error { + _, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + return nil +} + +func validateDirNotExists(dir string) error { + _, err := ioutil.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + return fmt.Errorf("dir %q still exists", dir) +} + +func validateFileExists(file string) error { + if _, err := os.Stat(file); err != nil { + return err + } + return nil } diff --git a/pkg/volume/util/subpath/subpath_nsenter.go b/pkg/volume/util/subpath/subpath_nsenter.go index ce5d825da79..b24379adbbf 100644 --- a/pkg/volume/util/subpath/subpath_nsenter.go +++ b/pkg/volume/util/subpath/subpath_nsenter.go @@ -16,295 +16,48 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( "fmt" "os" "path/filepath" - "strings" "syscall" "golang.org/x/sys/unix" + "k8s.io/klog" "k8s.io/utils/nsenter" - utilpath "k8s.io/utils/path" + + "k8s.io/kubernetes/pkg/util/mount" ) -const ( - // hostProcMountsPath is the default mount path for rootfs - hostProcMountsPath = "/rootfs/proc/1/mounts" - // hostProcMountinfoPath is the default mount info path for rootfs - hostProcMountinfoPath = "/rootfs/proc/1/mountinfo" -) - -// Currently, all docker containers receive their own mount namespaces. -// NsenterMounter works by executing nsenter to run commands in -// the host's mount namespace. -type NsenterMounter struct { - ne *nsenter.Nsenter - // rootDir is location of /var/lib/kubelet directory. +type subpathNSE struct { + mounter mount.Interface + ne *nsenter.Nsenter rootDir string } -// NewNsenterMounter creates a new mounter for kubelet that runs as a container. -func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter { - return &NsenterMounter{ - rootDir: rootDir, +// Compile time-check for all implementers of subpath interface +var _ Interface = &subpathNSE{} + +// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter +// It is only valid on Linux systems +func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface { + return &subpathNSE{ + mounter: mounter, ne: ne, + rootDir: rootDir, } } -// NsenterMounter implements mount.Interface -var _ = Interface(&NsenterMounter{}) - -// Mount runs mount(8) in the host's root mount namespace. Aside from this -// aspect, Mount has the same semantics as the mounter returned by mount.New() -func (n *NsenterMounter) Mount(source string, target string, fstype string, options []string) error { - bind, bindOpts, bindRemountOpts := isBind(options) - - if bind { - err := n.doNsenterMount(source, target, fstype, bindOpts) - if err != nil { - return err - } - return n.doNsenterMount(source, target, fstype, bindRemountOpts) - } - - return n.doNsenterMount(source, target, fstype, options) +func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(sp.mounter, podDir, volumeName) } -// doNsenterMount nsenters the host's mount namespace and performs the -// requested mount. -func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error { - klog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options) - cmd, args := n.makeNsenterArgs(source, target, fstype, options) - outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput() - if len(outputBytes) != 0 { - klog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes)) - } - return err -} - -// makeNsenterArgs makes a list of argument to nsenter in order to do the -// requested mount. -func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) (string, []string) { - mountCmd := n.ne.AbsHostPath("mount") - mountArgs := makeMountArgs(source, target, fstype, options) - - if systemdRunPath, hasSystemd := n.ne.SupportsSystemd(); hasSystemd { - // Complete command line: - // nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/systemd-run --description=... --scope -- /bin/mount -t - // Expected flow is: - // * nsenter breaks out of container's mount namespace and executes - // host's systemd-run. - // * systemd-run creates a transient scope (=~ cgroup) and executes its - // argument (/bin/mount) there. - // * mount does its job, forks a fuse daemon if necessary and finishes. - // (systemd-run --scope finishes at this point, returning mount's exit - // code and stdout/stderr - thats one of --scope benefits). - // * systemd keeps the fuse daemon running in the scope (i.e. in its own - // cgroup) until the fuse daemon dies (another --scope benefit). - // Kubelet container can be restarted and the fuse daemon survives. - // * When the daemon dies (e.g. during unmount) systemd removes the - // scope automatically. - mountCmd, mountArgs = addSystemdScope(systemdRunPath, target, mountCmd, mountArgs) - } else { - // Fall back to simple mount when the host has no systemd. - // Complete command line: - // nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/mount -t - // Expected flow is: - // * nsenter breaks out of container's mount namespace and executes host's /bin/mount. - // * mount does its job, forks a fuse daemon if necessary and finishes. - // * Any fuse daemon runs in cgroup of kubelet docker container, - // restart of kubelet container will kill it! - - // No code here, mountCmd and mountArgs use /bin/mount - } - - return mountCmd, mountArgs -} - -// Unmount runs umount(8) in the host's mount namespace. -func (n *NsenterMounter) Unmount(target string) error { - args := []string{target} - // No need to execute systemd-run here, it's enough that unmount is executed - // in the host's mount namespace. It will finish appropriate fuse daemon(s) - // running in any scope. - klog.V(5).Infof("nsenter unmount args: %v", args) - outputBytes, err := n.ne.Exec("umount", args).CombinedOutput() - if len(outputBytes) != 0 { - klog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes)) - } - return err -} - -// List returns a list of all mounted filesystems in the host's mount namespace. -func (*NsenterMounter) List() ([]MountPoint, error) { - return listProcMounts(hostProcMountsPath) -} - -func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) { - return isNotMountPoint(m, dir) -} - -func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool { - deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) - return (mp.Path == dir) || (mp.Path == deletedDir) -} - -// IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt -// in the host's root mount namespace. -func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) { - file, err := filepath.Abs(file) - if err != nil { - return true, err - } - - // Check the directory exists - if _, err = os.Stat(file); os.IsNotExist(err) { - klog.V(5).Infof("findmnt: directory %s does not exist", file) - return true, err - } - - // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts - resolvedFile, err := n.EvalHostSymlinks(file) - if err != nil { - return true, err - } - - // Add --first-only option: since we are testing for the absence of a mountpoint, it is sufficient to get only - // the first of multiple possible mountpoints using --first-only. - // Also add fstype output to make sure that the output of target file will give the full path - // TODO: Need more refactoring for this function. Track the solution with issue #26996 - args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", resolvedFile} - klog.V(5).Infof("nsenter findmnt args: %v", args) - out, err := n.ne.Exec("findmnt", args).CombinedOutput() - if err != nil { - klog.V(2).Infof("Failed findmnt command for path %s: %s %v", resolvedFile, out, err) - // Different operating systems behave differently for paths which are not mount points. - // On older versions (e.g. 2.20.1) we'd get error, on newer ones (e.g. 2.26.2) we'd get "/". - // It's safer to assume that it's not a mount point. - return true, nil - } - mountTarget, err := parseFindMnt(string(out)) - if err != nil { - return false, err - } - - klog.V(5).Infof("IsLikelyNotMountPoint findmnt output for path %s: %v:", resolvedFile, mountTarget) - - if mountTarget == resolvedFile { - klog.V(5).Infof("IsLikelyNotMountPoint: %s is a mount point", resolvedFile) - return false, nil - } - klog.V(5).Infof("IsLikelyNotMountPoint: %s is not a mount point", resolvedFile) - return true, nil -} - -// parse output of "findmnt -o target,fstype" and return just the target -func parseFindMnt(out string) (string, error) { - // cut trailing newline - out = strings.TrimSuffix(out, "\n") - // cut everything after the last space - it's the filesystem type - i := strings.LastIndex(out, " ") - if i == -1 { - return "", fmt.Errorf("error parsing findmnt output, expected at least one space: %q", out) - } - return out[:i], nil -} - -// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. -// Returns true if open returns errno EBUSY, and false if errno is nil. -// Returns an error if errno is any error other than EBUSY. -// Returns with error if pathname is not a device. -func (n *NsenterMounter) DeviceOpened(pathname string) (bool, error) { - return exclusiveOpenFailsOnDevice(pathname) -} - -// PathIsDevice uses FileInfo returned from os.Stat to check if path refers -// to a device. -func (n *NsenterMounter) PathIsDevice(pathname string) (bool, error) { - pathType, err := n.GetFileType(pathname) - isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev - return isDevice, err -} - -//GetDeviceNameFromMount given a mount point, find the volume id from checking /proc/mounts -func (n *NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { - return getDeviceNameFromMount(n, mountPath, pluginDir) -} - -func (n *NsenterMounter) MakeRShared(path string) error { - return doMakeRShared(path, hostProcMountinfoPath) -} - -func (mounter *NsenterMounter) GetFileType(pathname string) (FileType, error) { - var pathType FileType - outputBytes, err := mounter.ne.Exec("stat", []string{"-L", "--printf=%F", pathname}).CombinedOutput() - if err != nil { - if strings.Contains(string(outputBytes), "No such file") { - err = fmt.Errorf("%s does not exist", pathname) - } else { - err = fmt.Errorf("stat %s error: %v", pathname, string(outputBytes)) - } - return pathType, err - } - - switch string(outputBytes) { - case "socket": - return FileTypeSocket, nil - case "character special file": - return FileTypeCharDev, nil - case "block special file": - return FileTypeBlockDev, nil - case "directory": - return FileTypeDirectory, nil - case "regular file": - return FileTypeFile, nil - } - - return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device") -} - -func (mounter *NsenterMounter) MakeDir(pathname string) error { - args := []string{"-p", pathname} - if _, err := mounter.ne.Exec("mkdir", args).CombinedOutput(); err != nil { - return err - } - return nil -} - -func (mounter *NsenterMounter) MakeFile(pathname string) error { - args := []string{pathname} - if _, err := mounter.ne.Exec("touch", args).CombinedOutput(); err != nil { - return err - } - return nil -} - -func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) { - // Resolve the symlinks but allow the target not to exist. EvalSymlinks - // would return an generic error when the target does not exist. - hostPath, err := mounter.ne.EvalSymlinks(pathname, false /* mustExist */) - if err != nil { - return false, err - } - kubeletpath := mounter.ne.KubeletPath(hostPath) - return utilpath.Exists(utilpath.CheckFollowSymlink, kubeletpath) -} - -func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error) { - return mounter.ne.EvalSymlinks(pathname, true) -} - -func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { - return doCleanSubPaths(mounter, podDir, volumeName) -} - -func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { +func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { // Bind-mount the subpath to avoid using symlinks in subpaths. - newHostPath, err = doNsEnterBindSubPath(mounter, subPath) + newHostPath, err = sp.doNsEnterBindSubPath(subPath) // There is no action when the container starts. Bind-mount will be cleaned // when container stops by CleanSubPaths. @@ -312,22 +65,22 @@ func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath return newHostPath, cleanupAction, err } -func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { +func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error { fullSubdirPath := filepath.Join(base, subdir) - evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */) + evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */) if err != nil { return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err) } evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath) - evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */) + evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */) if err != nil { return fmt.Errorf("error resolving symlinks in %s: %s", base, err) } evaluatedBase = filepath.Clean(evaluatedBase) - rootDir := filepath.Clean(mounter.rootDir) - if PathWithinBase(evaluatedBase, rootDir) { + rootDir := filepath.Clean(sp.rootDir) + if mount.PathWithinBase(evaluatedBase, rootDir) { // Base is in /var/lib/kubelet. This directory is shared between the // container with kubelet and the host. We don't need to add '/rootfs'. // This is useful when /rootfs is mounted as read-only - we can still @@ -338,27 +91,12 @@ func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.F // Base is somewhere on the host's filesystem. Add /rootfs and try to make // the directory there. // This requires /rootfs to be writable. - kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath) - kubeletBase := mounter.ne.KubeletPath(evaluatedBase) + kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath) + kubeletBase := sp.ne.KubeletPath(evaluatedBase) return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm) } -func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) { - exists, err := mounter.ExistsPath(pathname) - if err != nil { - return nil, err - } - if !exists { - return []string{}, nil - } - hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */) - if err != nil { - return nil, err - } - return searchMountPoints(hostpath, hostProcMountinfoPath) -} - -func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) { +func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) { // Linux, kubelet runs in a container: // - safely open the subpath // - bind-mount the subpath to target (this can be unsafe) @@ -367,26 +105,26 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st // control) // Evaluate all symlinks here once for all subsequent functions. - evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/) + evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/) if err != nil { return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) } - evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/) + evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/) if err != nil { return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) } klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath) - subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath) - subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath) + subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath) + subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath) // Check the subpath is correct and open it - fd, err := safeOpenSubPath(mounter, subpath) + fd, err := safeOpenSubPath(sp.mounter, subpath) if err != nil { return "", err } defer syscall.Close(fd) - alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) + alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath) if err != nil { return "", err } @@ -399,7 +137,7 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st // Cleanup subpath on error if !success { klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) - if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { + if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil { klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) } } @@ -409,7 +147,7 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st // expanded evalSubPath with evil symlinks and bind-mount it. // Mount is done on the host! don't use kubelet path! klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget) - if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { + if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err) } @@ -446,25 +184,3 @@ func checkDeviceInode(fd int, path string) error { } return nil } - -func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) { - hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */) - if err != nil { - return -1, err - } - kubeletpath := mounter.ne.KubeletPath(hostPath) - return getFSGroup(kubeletpath) -} - -func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) { - return getSELinuxSupport(pathname, hostProcMountsPath) -} - -func (mounter *NsenterMounter) GetMode(pathname string) (os.FileMode, error) { - hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */) - if err != nil { - return 0, err - } - kubeletpath := mounter.ne.KubeletPath(hostPath) - return getMode(kubeletpath) -} diff --git a/pkg/volume/util/subpath/subpath_nsenter_test.go b/pkg/volume/util/subpath/subpath_nsenter_test.go index 1ed4bd222b3..865d449f5c9 100644 --- a/pkg/volume/util/subpath/subpath_nsenter_test.go +++ b/pkg/volume/util/subpath/subpath_nsenter_test.go @@ -16,66 +16,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( "io/ioutil" "os" - "os/user" "path/filepath" "strings" "testing" "golang.org/x/sys/unix" + "k8s.io/utils/nsenter" + + "k8s.io/kubernetes/pkg/util/mount" ) -func TestParseFindMnt(t *testing.T) { - tests := []struct { - input string - target string - expectError bool - }{ - { - // standard mount name, e.g. for AWS - "/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389 ext4\n", - "/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389", - false, - }, - { - // mount name with space, e.g. vSphere - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n", - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk", - false, - }, - { - // hypotetic mount with several spaces - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n", - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk", - false, - }, - { - // invalid output - no filesystem type - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/blabla", - "", - true, - }, - } - - for i, test := range tests { - target, err := parseFindMnt(test.input) - if test.expectError && err == nil { - t.Errorf("test %d expected error, got nil", i) - } - if !test.expectError && err != nil { - t.Errorf("test %d returned error: %s", i, err) - } - if target != test.target { - t.Errorf("test %d expected %q, got %q", i, test.target, target) - } - } -} - func TestCheckDeviceInode(t *testing.T) { testDir, err := ioutil.TempDir("", "nsenter-mounter-device-") if err != nil { @@ -150,359 +106,22 @@ func TestCheckDeviceInode(t *testing.T) { } } -func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) { - rootfsPath = filepath.Join(tmpdir, "rootfs") +func newFakeNsenterMounter(tmpdir string, t *testing.T) (*mount.NsenterMounter, string, string, *nsenter.Nsenter, error) { + rootfsPath := filepath.Join(tmpdir, "rootfs") if err := os.Mkdir(rootfsPath, 0755); err != nil { - return nil, "", "", err + return nil, "", "", nil, err } ne, err := nsenter.NewFakeNsenter(rootfsPath) if err != nil { - return nil, "", "", err + return nil, "", "", nil, err } - varlibPath = filepath.Join(tmpdir, "/var/lib/kubelet") + varlibPath := filepath.Join(tmpdir, "/var/lib/kubelet") if err := os.MkdirAll(varlibPath, 0755); err != nil { - return nil, "", "", err + return nil, "", "", nil, err } - return NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, nil -} - -func TestNsenterExistsFile(t *testing.T) { - var isRoot bool - usr, err := user.Current() - if err == nil { - isRoot = usr.Username == "root" - } else { - switch err.(type) { - case user.UnknownUserIdError: - // Root should be always known, this is some random UID - isRoot = false - default: - t.Fatal(err) - } - } - - tests := []struct { - name string - prepare func(base, rootfs string) (string, error) - expectedOutput bool - expectError bool - }{ - { - name: "simple existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/file - path := filepath.Join(base, "file") - if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { - return "", err - } - // In kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, path, 0644); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: true, - }, - { - name: "simple non-existing file", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "file") - return path, nil - }, - expectedOutput: false, - }, - { - name: "simple non-accessible file", - prepare: func(base, rootfs string) (string, error) { - // On the host: - // create /base/dir/file, then make the dir inaccessible - dir := filepath.Join(base, "dir") - if err := os.MkdirAll(dir, 0755); err != nil { - return "", err - } - path := filepath.Join(dir, "file") - if err := ioutil.WriteFile(path, []byte{}, 0); err != nil { - return "", err - } - if err := os.Chmod(dir, 0644); err != nil { - return "", err - } - - // In kubelet: do the same with /rootfs/base/dir/file - rootfsPath, err := writeRootfsFile(rootfs, path, 0777) - if err != nil { - return "", err - } - rootfsDir := filepath.Dir(rootfsPath) - if err := os.Chmod(rootfsDir, 0644); err != nil { - return "", err - } - - return path, nil - }, - expectedOutput: isRoot, // ExistsPath success when running as root - expectError: !isRoot, // ExistsPath must fail when running as not-root - }, - { - name: "relative symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink("file", path); err != nil { - return "", err - } - // In kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: true, - }, - { - name: "absolute symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> /base/file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink(file, path); err != nil { - return "", err - } - // In kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { - return "", err - } - - return path, nil - }, - expectedOutput: true, - }, - { - name: "relative symlink to non-existing file", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "link") - if err := os.Symlink("file", path); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: false, - }, - { - name: "absolute symlink to non-existing file", - prepare: func(base, rootfs string) (string, error) { - file := filepath.Join(base, "file") - path := filepath.Join(base, "link") - if err := os.Symlink(file, path); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: false, - }, - { - name: "symlink loop", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "link") - if err := os.Symlink(path, path); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: false, - // TODO: realpath -m is not able to detect symlink loop. Should we care? - expectError: false, - }, - } - - for _, test := range tests { - tmpdir, err := ioutil.TempDir("", "nsenter-exists-file") - if err != nil { - t.Error(err) - continue - } - defer os.RemoveAll(tmpdir) - - testBase := filepath.Join(tmpdir, "base") - if err := os.Mkdir(testBase, 0755); err != nil { - t.Error(err) - continue - } - - mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t) - if err != nil { - t.Error(err) - continue - } - - path, err := test.prepare(testBase, rootfs) - if err != nil { - t.Error(err) - continue - } - - out, err := mounter.ExistsPath(path) - if err != nil && !test.expectError { - t.Errorf("Test %q: unexpected error: %s", test.name, err) - } - if err == nil && test.expectError { - t.Errorf("Test %q: expected error, got none", test.name) - } - - if out != test.expectedOutput { - t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedOutput, out) - } - } -} - -func TestNsenterGetMode(t *testing.T) { - tests := []struct { - name string - prepare func(base, rootfs string) (string, error) - expectedMode os.FileMode - expectError bool - }{ - { - name: "simple file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/file - path := filepath.Join(base, "file") - if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { - return "", err - } - - // Prepare a different file as /rootfs/base/file (="the host - // visible from container") to check that NsEnterMounter calls - // stat on this file and not on /base/file. - // Visible from kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, path, 0777); err != nil { - return "", err - } - - return path, nil - }, - expectedMode: 0777, - }, - { - name: "non-existing file", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "file") - return path, nil - }, - expectedMode: 0, - expectError: true, - }, - { - name: "absolute symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> /base/file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0644); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink(file, path); err != nil { - return "", err - } - - // Visible from kubelet: - // /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0747); err != nil { - return "", err - } - - return path, nil - }, - expectedMode: 0747, - }, - { - name: "relative symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0741); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink("file", path); err != nil { - return "", err - } - - // Visible from kubelet: - // /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0647); err != nil { - return "", err - } - - return path, nil - }, - expectedMode: 0647, - }, - } - - for _, test := range tests { - tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") - if err != nil { - t.Error(err) - continue - } - defer os.RemoveAll(tmpdir) - - testBase := filepath.Join(tmpdir, "base") - if err := os.Mkdir(testBase, 0755); err != nil { - t.Error(err) - continue - } - - mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t) - if err != nil { - t.Error(err) - continue - } - - path, err := test.prepare(testBase, rootfs) - if err != nil { - t.Error(err) - continue - } - - mode, err := mounter.GetMode(path) - if err != nil && !test.expectError { - t.Errorf("Test %q: unexpected error: %s", test.name, err) - } - if err == nil && test.expectError { - t.Errorf("Test %q: expected error, got none", test.name) - } - - if mode != test.expectedMode { - t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedMode, mode) - } - } -} - -func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) { - fullPath := filepath.Join(rootfs, path) - dir := filepath.Dir(fullPath) - if err := os.MkdirAll(dir, 0755); err != nil { - return "", err - } - if err := ioutil.WriteFile(fullPath, []byte{}, mode); err != nil { - return "", err - } - // Use chmod, io.WriteFile is affected by umask - if err := os.Chmod(fullPath, mode); err != nil { - return "", err - } - return fullPath, nil + return mount.NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, ne, nil } func TestNsenterSafeMakeDir(t *testing.T) { @@ -659,18 +278,21 @@ func TestNsenterSafeMakeDir(t *testing.T) { }, } for _, test := range tests { - tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") + tmpdir, err := ioutil.TempDir("", "nsenter-get-safedir-") if err != nil { t.Error(err) continue } defer os.RemoveAll(tmpdir) - mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t) + mounter, rootfs, varlib, ne, err := newFakeNsenterMounter(tmpdir, t) if err != nil { t.Error(err) continue } + + fsp := NewNSEnter(mounter, ne, varlib) + // Prepare base directory for the test testBase := filepath.Join(tmpdir, "base") if err := os.Mkdir(testBase, 0755); err != nil { @@ -700,7 +322,7 @@ func TestNsenterSafeMakeDir(t *testing.T) { testBase = varlib } - err = mounter.SafeMakeDir(test.subdir, testBase, 0755) + err = fsp.SafeMakeDir(test.subdir, testBase, 0755) if err != nil && !test.expectError { t.Errorf("Test %q: unexpected error: %s", test.name, err) } diff --git a/pkg/volume/util/subpath/subpath_unsupported.go b/pkg/volume/util/subpath/subpath_unsupported.go index 918d58cd247..a1fd600a286 100644 --- a/pkg/volume/util/subpath/subpath_unsupported.go +++ b/pkg/volume/util/subpath/subpath_unsupported.go @@ -16,124 +16,39 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( "errors" "os" + + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/utils/nsenter" ) -type Mounter struct { - mounterPath string +type subpath struct{} + +var errUnsupported = errors.New("util/subpath on this platform is not supported") + +// New returns a subpath.Interface for the current system. +func New(mount.Interface) Interface { + return &subpath{} } -var unsupportedErr = errors.New("util/mount on this platform is not supported") - -// New returns a mount.Interface for the current system. -// It provides options to override the default mounter behavior. -// mounterPath allows using an alternative to `/bin/mount` for mounting. -func New(mounterPath string) Interface { - return &Mounter{ - mounterPath: mounterPath, - } +// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all +// OS choices. however, NSEnter is only valid on Linux +func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface { + return nil } -func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { - return unsupportedErr +func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, errUnsupported } -func (mounter *Mounter) Unmount(target string) error { - return unsupportedErr +func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error { + return errUnsupported } -func (mounter *Mounter) List() ([]MountPoint, error) { - return []MountPoint{}, unsupportedErr -} - -func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { - return (mp.Path == dir) -} - -func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { - return isNotMountPoint(mounter, dir) -} - -func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - return true, unsupportedErr -} - -func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { - return "", unsupportedErr -} - -func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { - return "", unsupportedErr -} - -func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { - return false, unsupportedErr -} - -func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { - return true, unsupportedErr -} - -func (mounter *Mounter) MakeRShared(path string) error { - return unsupportedErr -} - -func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { - return mounter.Interface.Mount(source, target, fstype, options) -} - -func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { - return true, unsupportedErr -} - -func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { - return FileType("fake"), unsupportedErr -} - -func (mounter *Mounter) MakeDir(pathname string) error { - return unsupportedErr -} - -func (mounter *Mounter) MakeFile(pathname string) error { - return unsupportedErr -} - -func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { - return true, errors.New("not implemented") -} - -func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { - return "", unsupportedErr -} - -func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - return subPath.Path, nil, unsupportedErr -} - -func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { - return unsupportedErr -} - -func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { - return unsupportedErr -} - -func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { - return nil, errors.New("not implemented") -} - -func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { - return -1, errors.New("not implemented") -} - -func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { - return false, errors.New("not implemented") -} - -func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { - return 0, errors.New("not implemented") +func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return errUnsupported } diff --git a/pkg/volume/util/subpath/subpath_windows.go b/pkg/volume/util/subpath/subpath_windows.go index 917d0ce5114..2bbb3c52716 100644 --- a/pkg/volume/util/subpath/subpath_windows.go +++ b/pkg/volume/util/subpath/subpath_windows.go @@ -16,269 +16,33 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( "fmt" "os" - "os/exec" - "path" "path/filepath" - "strconv" "strings" "syscall" "k8s.io/klog" - - utilpath "k8s.io/utils/path" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/utils/nsenter" ) -// Mounter provides the default implementation of mount.Interface -// for the windows platform. This implementation assumes that the -// kubelet is running in the host's root mount namespace. -type Mounter struct { - mounterPath string +type subpath struct{} + +// New returns a subpath.Interface for the current system +func New(mount.Interface) Interface { + return &subpath{} } -// New returns a mount.Interface for the current system. -// It provides options to override the default mounter behavior. -// mounterPath allows using an alternative to `/bin/mount` for mounting. -func New(mounterPath string) Interface { - return &Mounter{ - mounterPath: mounterPath, - } -} - -// Mount : mounts source to target with given options. -// currently only supports cifs(smb), bind mount(for disk) -func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { - target = normalizeWindowsPath(target) - - if source == "tmpfs" { - klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, options) - return os.MkdirAll(target, 0755) - } - - parentDir := filepath.Dir(target) - if err := os.MkdirAll(parentDir, 0755); err != nil { - return err - } - - klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", - options, source, target, fstype) - bindSource := source - - // tell it's going to mount azure disk or azure file according to options - if bind, _, _ := isBind(options); bind { - // mount azure disk - bindSource = normalizeWindowsPath(source) - } else { - if len(options) < 2 { - klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", - options, len(options), source, target) - return nil - } - - // currently only cifs mount is supported - if strings.ToLower(fstype) != "cifs" { - return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options) - } - - if output, err := newSMBMapping(options[0], options[1], source); err != nil { - if isSMBMappingExist(source) { - klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source) - if output, err := removeSMBMapping(source); err != nil { - return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output) - } - if output, err := newSMBMapping(options[0], options[1], source); err != nil { - return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output) - } - } else { - return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output) - } - } - } - - if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil { - klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) - return err - } - +// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all +// OS choices. however, NSEnter is only valid on Linux +func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface { return nil } -// do the SMB mount with username, password, remotepath -// return (output, error) -func newSMBMapping(username, password, remotepath string) (string, error) { - if username == "" || password == "" || remotepath == "" { - return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, password, remotepath) - } - - // use PowerShell Environment Variables to store user input string to prevent command line injection - // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 - cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + - `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + - `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential` - cmd := exec.Command("powershell", "/c", cmdLine) - cmd.Env = append(os.Environ(), - fmt.Sprintf("smbuser=%s", username), - fmt.Sprintf("smbpassword=%s", password), - fmt.Sprintf("smbremotepath=%s", remotepath)) - - output, err := cmd.CombinedOutput() - return string(output), err -} - -// check whether remotepath is already mounted -func isSMBMappingExist(remotepath string) bool { - cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`) - cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) - _, err := cmd.CombinedOutput() - return err == nil -} - -// remove SMB mapping -func removeSMBMapping(remotepath string) (string, error) { - cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`) - cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) - output, err := cmd.CombinedOutput() - return string(output), err -} - -// Unmount unmounts the target. -func (mounter *Mounter) Unmount(target string) error { - klog.V(4).Infof("azureMount: Unmount target (%q)", target) - target = normalizeWindowsPath(target) - if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { - klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) - return err - } - return nil -} - -// List returns a list of all mounted filesystems. todo -func (mounter *Mounter) List() ([]MountPoint, error) { - return []MountPoint{}, nil -} - -// IsMountPointMatch determines if the mountpoint matches the dir -func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { - return mp.Path == dir -} - -// IsNotMountPoint determines if a directory is a mountpoint. -func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { - return isNotMountPoint(mounter, dir) -} - -// IsLikelyNotMountPoint determines if a directory is not a mountpoint. -func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - stat, err := os.Lstat(file) - if err != nil { - return true, err - } - // If current file is a symlink, then it is a mountpoint. - if stat.Mode()&os.ModeSymlink != 0 { - target, err := os.Readlink(file) - if err != nil { - return true, fmt.Errorf("readlink error: %v", err) - } - exists, err := mounter.ExistsPath(target) - if err != nil { - return true, err - } - return !exists, nil - } - - return true, nil -} - -// GetDeviceNameFromMount given a mnt point, find the device -func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { - return getDeviceNameFromMount(mounter, mountPath, pluginDir) -} - -// getDeviceNameFromMount find the device(drive) name in which -// the mount path reference should match the given plugin directory. In case no mount path reference -// matches, returns the volume name taken from its given mountPath -func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { - refs, err := mounter.GetMountRefs(mountPath) - if err != nil { - klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) - return "", err - } - if len(refs) == 0 { - return "", fmt.Errorf("directory %s is not mounted", mountPath) - } - basemountPath := normalizeWindowsPath(path.Join(pluginDir, MountsInGlobalPDPath)) - for _, ref := range refs { - if strings.Contains(ref, basemountPath) { - volumeID, err := filepath.Rel(normalizeWindowsPath(basemountPath), ref) - if err != nil { - klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) - return "", err - } - return volumeID, nil - } - } - - return path.Base(mountPath), nil -} - -// DeviceOpened determines if the device is in use elsewhere -func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { - return false, nil -} - -// PathIsDevice determines if a path is a device. -func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { - return false, nil -} - -// MakeRShared checks that given path is on a mount with 'rshared' mount -// propagation. Empty implementation here. -func (mounter *Mounter) MakeRShared(path string) error { - return nil -} - -// GetFileType checks for sockets/block/character devices -func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { - return getFileType(pathname) -} - -// MakeFile creates a new directory -func (mounter *Mounter) MakeDir(pathname string) error { - err := os.MkdirAll(pathname, os.FileMode(0755)) - if err != nil { - if !os.IsExist(err) { - return err - } - } - return nil -} - -// MakeFile creates an empty file -func (mounter *Mounter) MakeFile(pathname string) error { - f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) - defer f.Close() - if err != nil { - if !os.IsExist(err) { - return err - } - } - return nil -} - -// ExistsPath checks whether the path exists -func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { - return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) -} - -// EvalHostSymlinks returns the path name after evaluating symlinks -func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { - return filepath.EvalSymlinks(pathname) -} - // check whether hostPath is within volume path // this func will lock all intermediate subpath directories, need to close handle outside of this func after container started func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) { @@ -310,7 +74,7 @@ func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, e if err != nil { return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err) } - if startsWithBackstep(relSubPath) { + if mount.StartsWithBackstep(relSubPath) { return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath) } @@ -345,7 +109,7 @@ func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, e break } - if !PathWithinBase(currentFullPath, volumePath) { + if !mount.PathWithinBase(currentFullPath, volumePath) { errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath) break } @@ -381,7 +145,7 @@ func lockPath(path string) (uintptr, error) { } // Lock all directories in subPath and check they're not symlinks. -func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { +func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path) // Unlock the directories when the container starts @@ -392,139 +156,12 @@ func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, } // No bind-mounts for subpaths are necessary on Windows -func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { +func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error { return nil } -func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { - // Try to mount the disk - klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) - - if err := ValidateDiskNumber(source); err != nil { - klog.Errorf("diskMount: formatAndMount failed, err: %v", err) - return err - } - - if len(fstype) == 0 { - // Use 'NTFS' as the default - fstype = "NTFS" - } - - // format disk if it is unformatted(raw) - cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ - " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) - if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil { - return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) - } - klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) - - driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec) - if err != nil { - return err - } - driverPath := driveLetter + ":" - target = normalizeWindowsPath(target) - klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) - if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil { - klog.Errorf("mklink failed: %v, output: %q", err, string(output)) - return err - } - return nil -} - -func normalizeWindowsPath(path string) string { - normalizedPath := strings.Replace(path, "/", "\\", -1) - if strings.HasPrefix(normalizedPath, "\\") { - normalizedPath = "c:" + normalizedPath - } - return normalizedPath -} - -// ValidateDiskNumber : disk number should be a number in [0, 99] -func ValidateDiskNumber(disk string) error { - diskNum, err := strconv.Atoi(disk) - if err != nil { - return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err) - } - - if diskNum < 0 || diskNum > 99 { - return fmt.Errorf("disk number out of range: %q", disk) - } - - return nil -} - -// Get drive letter according to windows disk number -func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) { - cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum) - output, err := exec.Run("powershell", "/c", cmd) - if err != nil { - return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output)) - } - if len(string(output)) < 1 { - return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty") - } - return string(output)[:1], nil -} - -// getAllParentLinks walks all symbolic links and return all the parent targets recursively -func getAllParentLinks(path string) ([]string, error) { - const maxIter = 255 - links := []string{} - for { - links = append(links, path) - if len(links) > maxIter { - return links, fmt.Errorf("unexpected length of parent links: %v", links) - } - - fi, err := os.Lstat(path) - if err != nil { - return links, fmt.Errorf("Lstat: %v", err) - } - if fi.Mode()&os.ModeSymlink == 0 { - break - } - - path, err = os.Readlink(path) - if err != nil { - return links, fmt.Errorf("Readlink error: %v", err) - } - } - - return links, nil -} - -// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows -func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { - if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) { - return []string{}, nil - } else if err != nil { - return nil, err - } - return []string{pathname}, nil -} - -// Note that on windows, it always returns 0. We actually don't set FSGroup on -// windows platform, see SetVolumeOwnership implementation. -func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { - return 0, nil -} - -func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { - // Windows does not support SELinux. - return false, nil -} - -func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { - info, err := os.Stat(pathname) - if err != nil { - return 0, err - } - return info.Mode(), nil -} - // SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. -func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { +func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error { realBase, err := filepath.EvalSymlinks(base) if err != nil { return fmt.Errorf("error resolving symlinks in %s: %s", base, err) @@ -537,7 +174,7 @@ func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { klog.V(4).Infof("Creating directory %q within base %q", pathname, base) - if !PathWithinBase(pathname, base) { + if !mount.PathWithinBase(pathname, base) { return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) } @@ -572,7 +209,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { if err != nil { return fmt.Errorf("cannot read link %s: %s", base, err) } - if !PathWithinBase(fullExistingPath, fullBasePath) { + if !mount.PathWithinBase(fullExistingPath, fullBasePath) { return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) } @@ -619,7 +256,7 @@ func findExistingPrefix(base, pathname string) (string, []string, error) { return base, nil, err } - if startsWithBackstep(rel) { + if mount.StartsWithBackstep(rel) { return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base) } diff --git a/pkg/volume/util/subpath/subpath_windows_test.go b/pkg/volume/util/subpath/subpath_windows_test.go index e918f3f9efa..82a1034f54b 100644 --- a/pkg/volume/util/subpath/subpath_windows_test.go +++ b/pkg/volume/util/subpath/subpath_windows_test.go @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( "fmt" @@ -24,54 +24,11 @@ import ( "os" "os/exec" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/assert" ) -func TestNormalizeWindowsPath(t *testing.T) { - path := `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4/volumes/kubernetes.io~azure-disk` - normalizedPath := normalizeWindowsPath(path) - if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` { - t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) - } - - path = `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` - normalizedPath = normalizeWindowsPath(path) - if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` { - t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) - } - - path = `/` - normalizedPath = normalizeWindowsPath(path) - if normalizedPath != `c:\` { - t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) - } -} - -func TestValidateDiskNumber(t *testing.T) { - diskNum := "0" - if err := ValidateDiskNumber(diskNum); err != nil { - t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) - } - - diskNum = "99" - if err := ValidateDiskNumber(diskNum); err != nil { - t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) - } - - diskNum = "ab" - if err := ValidateDiskNumber(diskNum); err == nil { - t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) - } - - diskNum = "100" - if err := ValidateDiskNumber(diskNum); err == nil { - t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) - } -} - func makeLink(link, target string) error { if output, err := exec.Command("cmd", "/c", "mklink", "/D", link, target).CombinedOutput(); err != nil { return fmt.Errorf("mklink failed: %v, link(%q) target(%q) output: %q", err, link, target, string(output)) @@ -79,61 +36,6 @@ func makeLink(link, target string) error { return nil } -func removeLink(link string) error { - if output, err := exec.Command("cmd", "/c", "rmdir", link).CombinedOutput(); err != nil { - return fmt.Errorf("rmdir failed: %v, output: %q", err, string(output)) - } - return nil -} - -func setEquivalent(set1, set2 []string) bool { - map1 := make(map[string]bool) - map2 := make(map[string]bool) - for _, s := range set1 { - map1[s] = true - } - for _, s := range set2 { - map2[s] = true - } - - for s := range map1 { - if !map2[s] { - return false - } - } - for s := range map2 { - if !map1[s] { - return false - } - } - return true -} - -// this func must run in admin mode, otherwise it will fail -func TestGetMountRefs(t *testing.T) { - tests := []struct { - mountPath string - expectedRefs []string - }{ - { - mountPath: `c:\windows`, - expectedRefs: []string{`c:\windows`}, - }, - { - mountPath: `c:\doesnotexist`, - expectedRefs: []string{}, - }, - } - - mounter := Mounter{"fake/path"} - - for _, test := range tests { - if refs, err := mounter.GetMountRefs(test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { - t.Errorf("getMountRefs(%q) = %v, error: %v; expected %v", test.mountPath, refs, err, test.expectedRefs) - } - } -} - func TestDoSafeMakeDir(t *testing.T) { base, err := ioutil.TempDir("", "TestDoSafeMakeDir") if err != nil { @@ -536,292 +438,3 @@ func TestFindExistingPrefix(t *testing.T) { // remove dir will happen after closing all file handles assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath) } - -func TestPathWithinBase(t *testing.T) { - tests := []struct { - fullPath string - basePath string - expectedResult bool - }{ - { - fullPath: `c:\tmp\a\b\c`, - basePath: `c:\tmp`, - expectedResult: true, - }, - { - fullPath: `c:\tmp1`, - basePath: `c:\tmp2`, - expectedResult: false, - }, - { - fullPath: `c:\tmp`, - basePath: `c:\tmp`, - expectedResult: true, - }, - { - fullPath: `c:\tmp`, - basePath: `c:\tmp\a\b\c`, - expectedResult: false, - }, - { - fullPath: `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config\..timestamp\file.txt`, - basePath: `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config`, - expectedResult: true, - }, - } - - for _, test := range tests { - result := PathWithinBase(test.fullPath, test.basePath) - assert.Equal(t, result, test.expectedResult, "Expect result not equal with PathWithinBase(%s, %s) return: %q, expected: %q", - test.fullPath, test.basePath, result, test.expectedResult) - } -} - -func TestGetFileType(t *testing.T) { - mounter := New("fake/path") - - testCase := []struct { - name string - expectedType FileType - setUp func() (string, string, error) - }{ - { - "Directory Test", - FileTypeDirectory, - func() (string, string, error) { - tempDir, err := ioutil.TempDir("", "test-get-filetype-") - return tempDir, tempDir, err - }, - }, - { - "File Test", - FileTypeFile, - func() (string, string, error) { - tempFile, err := ioutil.TempFile("", "test-get-filetype") - if err != nil { - return "", "", err - } - tempFile.Close() - return tempFile.Name(), tempFile.Name(), nil - }, - }, - } - - for idx, tc := range testCase { - path, cleanUpPath, err := tc.setUp() - if err != nil { - t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) - } - if len(cleanUpPath) > 0 { - defer os.RemoveAll(cleanUpPath) - } - - fileType, err := mounter.GetFileType(path) - if err != nil { - t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) - } - if fileType != tc.expectedType { - t.Fatalf("[%d-%s] expected %s, but got %s", idx, tc.name, tc.expectedType, fileType) - } - } -} - -func TestIsLikelyNotMountPoint(t *testing.T) { - mounter := Mounter{"fake/path"} - - tests := []struct { - fileName string - targetLinkName string - setUp func(base, fileName, targetLinkName string) error - expectedResult bool - expectError bool - }{ - { - "Dir", - "", - func(base, fileName, targetLinkName string) error { - return os.Mkdir(filepath.Join(base, fileName), 0750) - }, - true, - false, - }, - { - "InvalidDir", - "", - func(base, fileName, targetLinkName string) error { - return nil - }, - true, - true, - }, - { - "ValidSymLink", - "targetSymLink", - func(base, fileName, targetLinkName string) error { - targeLinkPath := filepath.Join(base, targetLinkName) - if err := os.Mkdir(targeLinkPath, 0750); err != nil { - return err - } - - filePath := filepath.Join(base, fileName) - if err := makeLink(filePath, targeLinkPath); err != nil { - return err - } - return nil - }, - false, - false, - }, - { - "InvalidSymLink", - "targetSymLink2", - func(base, fileName, targetLinkName string) error { - targeLinkPath := filepath.Join(base, targetLinkName) - if err := os.Mkdir(targeLinkPath, 0750); err != nil { - return err - } - - filePath := filepath.Join(base, fileName) - if err := makeLink(filePath, targeLinkPath); err != nil { - return err - } - return removeLink(targeLinkPath) - }, - true, - false, - }, - } - - for _, test := range tests { - base, err := ioutil.TempDir("", test.fileName) - if err != nil { - t.Fatalf(err.Error()) - } - - defer os.RemoveAll(base) - - if err := test.setUp(base, test.fileName, test.targetLinkName); err != nil { - t.Fatalf("unexpected error in setUp(%s, %s): %v", test.fileName, test.targetLinkName, err) - } - - filePath := filepath.Join(base, test.fileName) - result, err := mounter.IsLikelyNotMountPoint(filePath) - assert.Equal(t, result, test.expectedResult, "Expect result not equal with IsLikelyNotMountPoint(%s) return: %q, expected: %q", - filePath, result, test.expectedResult) - - if test.expectError { - assert.NotNil(t, err, "Expect error during IsLikelyNotMountPoint(%s)", filePath) - } else { - assert.Nil(t, err, "Expect error is nil during IsLikelyNotMountPoint(%s)", filePath) - } - } -} - -func TestFormatAndMount(t *testing.T) { - fakeMounter := ErrorMounter{&FakeMounter{}, 0, nil} - execCallback := func(cmd string, args ...string) ([]byte, error) { - for j := range args { - if strings.Contains(args[j], "Get-Disk -Number") { - return []byte("0"), nil - } - - if strings.Contains(args[j], "Get-Partition -DiskNumber") { - return []byte("0"), nil - } - - if strings.Contains(args[j], "mklink") { - return nil, nil - } - } - return nil, fmt.Errorf("Unexpected cmd %s, args %v", cmd, args) - } - fakeExec := NewFakeExec(execCallback) - - mounter := SafeFormatAndMount{ - Interface: &fakeMounter, - Exec: fakeExec, - } - - tests := []struct { - device string - target string - fstype string - mountOptions []string - expectError bool - }{ - { - "0", - "disk", - "NTFS", - []string{}, - false, - }, - { - "0", - "disk", - "", - []string{}, - false, - }, - { - "invalidDevice", - "disk", - "NTFS", - []string{}, - true, - }, - } - - for _, test := range tests { - base, err := ioutil.TempDir("", test.device) - if err != nil { - t.Fatalf(err.Error()) - } - defer os.RemoveAll(base) - - target := filepath.Join(base, test.target) - err = mounter.FormatAndMount(test.device, target, test.fstype, test.mountOptions) - if test.expectError { - assert.NotNil(t, err, "Expect error during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions) - } else { - assert.Nil(t, err, "Expect error is nil during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions) - } - } -} - -func TestNewSMBMapping(t *testing.T) { - tests := []struct { - username string - password string - remotepath string - expectError bool - }{ - { - "", - "password", - `\\remotepath`, - true, - }, - { - "username", - "", - `\\remotepath`, - true, - }, - { - "username", - "password", - "", - true, - }, - } - - for _, test := range tests { - _, err := newSMBMapping(test.username, test.password, test.remotepath) - if test.expectError { - assert.NotNil(t, err, "Expect error during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath) - } else { - assert.Nil(t, err, "Expect error is nil during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath) - } - } -}