github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor subpath out of pkg/util/mount

Browse Source 
This patch moves subpath functionality out of pkg/util/mount and into a
new package pkg/volume/util/subpath. NSEnter funtionality is maintained.
This commit is contained in:
Travis Rhoden 2019-02-26 12:29:27 -07:00
parent 766cf26897
commit 2c4d748bed
48 changed files with 467 additions and 6097 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kubelet/app/BUILD
View File

@ -108,6 +108,7 @@ go_library(
"//pkg/volume/scaleio:go_default_library",
"//pkg/volume/secret:go_default_library",
"//pkg/volume/storageos:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//pkg/volume/vsphere_volume:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",

5
cmd/kubelet/app/server.go
View File

@ -95,6 +95,7 @@ import (
"k8s.io/kubernetes/pkg/util/rlimit"
"k8s.io/kubernetes/pkg/version"
"k8s.io/kubernetes/pkg/version/verflag"
"k8s.io/kubernetes/pkg/volume/util/subpath"
nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
"k8s.io/utils/exec"
"k8s.io/utils/nsenter"
@ -364,6 +365,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
}
mounter := mount.New(s.ExperimentalMounterPath)
subpather := subpath.New(mounter)
var pluginRunner = exec.New()
if s.Containerized {
klog.V(2).Info("Running kubelet in containerized mode")
@ -372,6 +374,8 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
return nil, err
}
mounter = mount.NewNsenterMounter(s.RootDirectory, ne)
// NSenter only valid on Linux
subpather = subpath.NewNSEnter(mounter, ne, s.RootDirectory)
// an exec interface which can use nsenter for flex plugin calls
pluginRunner, err = nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
if err != nil {
@ -399,6 +403,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
CSIClient: nil,
EventClient: nil,
Mounter: mounter,
Subpather: subpather,
OOMAdjuster: oom.NewOOMAdjuster(),
OSInterface: kubecontainer.RealOS{},
VolumePlugins: ProbeVolumePlugins(),

2
pkg/controller/.import-restrictions
View File

@ -167,6 +167,7 @@
"AllowedPrefixes": [
"k8s.io/kubernetes/pkg/api/legacyscheme",
"k8s.io/kubernetes/pkg/api/v1/endpoints",
"k8s.io/kubernetes/pkg/api/v1/node",
"k8s.io/kubernetes/pkg/api/v1/pod",
"k8s.io/kubernetes/pkg/apis/apps/v1",
"k8s.io/kubernetes/pkg/apis/autoscaling",
@ -241,6 +242,7 @@
"k8s.io/kubernetes/pkg/volume/util",
"k8s.io/kubernetes/pkg/volume/util/operationexecutor",
"k8s.io/kubernetes/pkg/volume/util/recyclerclient",
"k8s.io/kubernetes/pkg/volume/util/subpath",
"k8s.io/kubernetes/pkg/volume/util/types",
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler",
"k8s.io/kubernetes/pkg/api/service",

1
pkg/controller/volume/attachdetach/BUILD
View File

@ -22,6 +22,7 @@ go_library(
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/operationexecutor:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",

6
pkg/controller/volume/attachdetach/attach_detach_controller.go
View File

@ -52,6 +52,7 @@ import (
"k8s.io/kubernetes/pkg/volume"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
)
@ -768,3 +769,8 @@ func (adc *attachDetachController) GetEventRecorder() record.EventRecorder {
func (adc *attachDetachController) GetCSIClient() csiclient.Interface {
return adc.csiClient
}
func (adc *attachDetachController) GetSubpather() subpath.Interface {
// Subpaths not needed in attachdetach controller
return nil
}

1
pkg/controller/volume/expand/BUILD
View File

@ -19,6 +19,7 @@ go_library(
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/operationexecutor:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",

6
pkg/controller/volume/expand/expand_controller.go
View File

@ -46,6 +46,7 @@ import (
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
)
@ -339,3 +340,8 @@ func (expc *expandController) GetCSIClient() csiclientset.Interface {
// No volume plugin in expand controller needs csi.storage.k8s.io
return nil
}
func (expc *expandController) GetSubpather() subpath.Interface {
// not needed for expand controller
return nil
}

1
pkg/controller/volume/persistentvolume/BUILD
View File

@ -33,6 +33,7 @@ go_library(
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/recyclerclient:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/storage/v1:go_default_library",

6
pkg/controller/volume/persistentvolume/volume_host.go
View File

@ -30,6 +30,7 @@ import (
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
vol "k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
// VolumeHost interface implementation for PersistentVolumeController.
@ -136,3 +137,8 @@ func (ctrl *PersistentVolumeController) GetCSIClient() csiclientset.Interface {
// No volume plugin needs csi.storage.k8s.io client in PV controller.
return nil
}
func (ctrl *PersistentVolumeController) GetSubpather() subpath.Interface {
// No volume plugin needs Subpaths in PV controller.
return nil
}

2
pkg/kubelet/BUILD
View File

@ -112,6 +112,7 @@ go_library(
"//pkg/volume:go_default_library",
"//pkg/volume/csi:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//pkg/volume/util/types:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
"//pkg/volume/validation:go_default_library",
@ -220,6 +221,7 @@ go_test(
"//pkg/volume/host_path:go_default_library",
"//pkg/volume/testing:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

6
pkg/kubelet/kubelet.go
View File

@ -113,6 +113,7 @@ import (
"k8s.io/kubernetes/pkg/util/oom"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/csi"
"k8s.io/kubernetes/pkg/volume/util/subpath"
nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned"
utilexec "k8s.io/utils/exec"
"k8s.io/utils/integer"
@ -255,6 +256,7 @@ type Dependencies struct {
OSInterface kubecontainer.OSInterface
PodConfig *config.PodConfig
Recorder record.EventRecorder
Subpather subpath.Interface
VolumePlugins []volume.VolumePlugin
DynamicPluginProber volume.DynamicPluginProber
TLSOptions *server.TLSOptions
@ -519,6 +521,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
cgroupsPerQOS: kubeCfg.CgroupsPerQOS,
cgroupRoot: kubeCfg.CgroupRoot,
mounter: kubeDeps.Mounter,
subpather: kubeDeps.Subpather,
maxPods: int(kubeCfg.MaxPods),
podsPerCore: int(kubeCfg.PodsPerCore),
syncLoopMonitor: atomic.Value{},
@ -1099,6 +1102,9 @@ type Kubelet struct {
// Mounter to use for volumes.
mounter mount.Interface
// subpather to execute subpath actions
subpather subpath.Interface
// Manager of non-Runtime containers.
containerManager cm.ContainerManager

9
pkg/kubelet/kubelet_pods.go
View File

@ -61,6 +61,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format"
mountutil "k8s.io/kubernetes/pkg/util/mount"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
"k8s.io/kubernetes/third_party/forked/golang/expansion"
@ -127,7 +128,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol
}
// makeMounts determines the mount points for the given container.
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, subpather subpath.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
// Kubernetes only mounts on /etc/hosts if:
// - container is not an infrastructure (pause) container
// - container is not already mounting on /etc/hosts
@ -206,13 +207,13 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
if err != nil {
return nil, cleanupAction, err
}
if err := mounter.SafeMakeDir(subPath, volumePath, perm); err != nil {
if err := subpather.SafeMakeDir(subPath, volumePath, perm); err != nil {
// Don't pass detailed error back to the user because it could give information about host filesystem
klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)
}
}
hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
hostPath, cleanupAction, err = subpather.PrepareSafeSubpath(subpath.Subpath{
VolumeMountIndex: i,
Path: hostPath,
VolumeName: vol.InnerVolumeSpecName,
@ -464,7 +465,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
}
opts.Envs = append(opts.Envs, envs...)
mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, opts.Envs)
mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, kl.subpather, opts.Envs)
if err != nil {
return nil, cleanupAction, err
}

4
pkg/kubelet/kubelet_pods_linux_test.go
View File

@ -29,6 +29,7 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/util/mount"
volumetest "k8s.io/kubernetes/pkg/volume/testing"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
func TestMakeMounts(t *testing.T) {
@ -241,13 +242,14 @@ func TestMakeMounts(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
fm := &mount.FakeMounter{}
fsp := &subpath.FakeSubpath{}
pod := v1.Pod{
Spec: v1.PodSpec{
HostNetwork: true,
},
}
mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, nil)
mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, fsp, nil)
// validate only the error if we expect an error
if tc.expectErr {

4
pkg/kubelet/kubelet_pods_test.go
View File

@ -49,10 +49,12 @@ import (
"k8s.io/kubernetes/pkg/kubelet/server/portforward"
"k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
func TestDisabledSubpath(t *testing.T) {
fm := &mount.FakeMounter{}
fsp := &subpath.FakeSubpath{}
pod := v1.Pod{
Spec: v1.PodSpec{
HostNetwork: true,
@ -95,7 +97,7 @@ func TestDisabledSubpath(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, false)()
for name, test := range cases {
_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, nil)
_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, fsp, nil)
if err != nil && !test.expectError {
t.Errorf("test %v failed: %v", name, err)
}

4
pkg/kubelet/kubelet_pods_windows_test.go
View File

@ -23,6 +23,7 @@ import (
"k8s.io/api/core/v1"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
func TestMakeMountsWindows(t *testing.T) {
@ -82,7 +83,8 @@ func TestMakeMountsWindows(t *testing.T) {
}
fm := &mount.FakeMounter{}
mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm, nil)
fsp := &subpath.FakeSubpath{}
mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm, fsp, nil)
expectedMounts := []kubecontainer.Mount{
{

2
pkg/kubelet/kubelet_test.go
View File

@ -75,6 +75,7 @@ import (
_ "k8s.io/kubernetes/pkg/volume/host_path"
volumetest "k8s.io/kubernetes/pkg/volume/testing"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
func init() {
@ -176,6 +177,7 @@ func newTestKubeletWithImageList(
kubelet.heartbeatClient = fakeKubeClient
kubelet.os = &containertest.FakeOS{}
kubelet.mounter = &mount.FakeMounter{}
kubelet.subpather = &subpath.FakeSubpath{}
kubelet.hostname = testKubeletHostname
kubelet.nodeName = types.NodeName(testKubeletHostname)

5
pkg/kubelet/volume_host.go
View File

@ -40,6 +40,7 @@ import (
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
// NewInitializedVolumePluginMgr returns a new instance of
@ -126,6 +127,10 @@ func (kvh *kubeletVolumeHost) GetCSIClient() csiclientset.Interface {
return kvh.kubelet.csiClient
}
func (kvh *kubeletVolumeHost) GetSubpather() subpath.Interface {
return kvh.kubelet.subpather
}
func (kvh *kubeletVolumeHost) NewWrapperMounter(
volName string,
spec volume.Spec,

1
pkg/kubemark/BUILD
View File

@ -36,6 +36,7 @@ go_library(
"//pkg/volume/emptydir:go_default_library",
"//pkg/volume/projected:go_default_library",
"//pkg/volume/secret:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",

2
pkg/kubemark/hollow_kubelet.go
View File

@ -35,6 +35,7 @@ import (
"k8s.io/kubernetes/pkg/volume/emptydir"
"k8s.io/kubernetes/pkg/volume/projected"
"k8s.io/kubernetes/pkg/volume/secret"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/kubernetes/test/utils"
"k8s.io/klog"
@ -78,6 +79,7 @@ func NewHollowKubelet(
TLSOptions: nil,
OOMAdjuster: oom.NewFakeOOMAdjuster(),
Mounter: mount.New("" /* default mount path */),
Subpather: &subpath.FakeSubpath{},
}
return &HollowKubelet{

2
pkg/util/mount/BUILD
View File

@ -80,8 +80,6 @@ go_test(
"//vendor/k8s.io/utils/exec/testing:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],

12
pkg/util/mount/exec_mount.go
View File

@ -144,18 +144,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return m.wrappedMounter.EvalHostSymlinks(pathname)
}
func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return m.wrappedMounter.PrepareSafeSubpath(subPath)
}
func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
}
func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
}
func (m *execMounter) GetMountRefs(pathname string) ([]string, error) {
return m.wrappedMounter.GetMountRefs(pathname)
}

12
pkg/util/mount/exec_mount_unsupported.go
View File

@ -91,18 +91,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

11
pkg/util/mount/fake.go
View File

@ -220,17 +220,6 @@ func (f *FakeMounter) EvalHostSymlinks(pathname string) (string, error) {
return pathname, nil
}
func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {

29
pkg/util/mount/mount.go
View File

@ -84,35 +84,12 @@ type Interface interface {
// MakeDir creates a new directory.
// Will operate in the host mount namespace if kubelet is running in a container
MakeDir(pathname string) error
// SafeMakeDir creates subdir within given base. It makes sure that the
// created directory does not escape given base directory mis-using
// symlinks. Note that the function makes sure that it creates the directory
// somewhere under the base, nothing else. E.g. if the directory already
// exists, it may exist outside of the base due to symlinks.
// This method should be used if the directory to create is inside volume
// that's under user control. User must not be able to use symlinks to
// escape the volume to create directories somewhere else.
SafeMakeDir(subdir string, base string, perm os.FileMode) error
// Will operate in the host mount namespace if kubelet is running in a container.
// Error is returned on any other error than "file not found".
ExistsPath(pathname string) (bool, error)
// EvalHostSymlinks returns the path name after evaluating symlinks.
// Will operate in the host mount namespace if kubelet is running in a container.
EvalHostSymlinks(pathname string) (string, error)
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(podDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
// GetMountRefs finds all mount references to the path, returns a
// list of paths. Path could be a mountpoint path, device or a normal
// directory (for bind mount).
@ -355,15 +332,15 @@ func PathWithinBase(fullPath, basePath string) bool {
if err != nil {
return false
}
if startsWithBackstep(rel) {
if StartsWithBackstep(rel) {
// Needed to escape the base path
return false
}
return true
}
// startsWithBackstep checks if the given path starts with a backstep segment
func startsWithBackstep(rel string) bool {
// StartsWithBackstep checks if the given path starts with a backstep segment
func StartsWithBackstep(rel string) bool {
// normalize to / and check for ../
return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
}

516
pkg/util/mount/mount_linux.go
View File

@ -21,7 +21,6 @@ package mount
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
@ -53,14 +52,6 @@ const (
fsckErrorsCorrected = 1
// 'fsck' found errors but exited without correcting them
fsckErrorsUncorrected = 4
// place for subpath mounts
// TODO: pass in directory using kubelet_getters instead
containerSubPathDirectoryName = "volume-subpaths"
// syscall.Openat flags used to traverse directories not following symlinks
nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
// flags for getting file descriptor without following the symlink
openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
)
// Mounter provides the default implementation of mount.Interface
@ -726,282 +717,6 @@ func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
return false, nil
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnterMounter
func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
if !PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
}
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
if err != nil {
return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
}
return fd, nil
}
// prepareSubpathTarget creates target for bind-mount of subpath. It returns
// "true" when the target already exists and something is mounted there.
// Given Subpath must have all paths with already resolved symlinks and with
// paths relevant to kubelet (when it runs in a container).
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := mounter.IsNotMountPoint(bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
return true, bindPathTarget, nil
}
// bindPathTarget is in /var/lib/kubelet and thus reachable without any
// translation even to containerized kubelet.
bindParent := filepath.Dir(bindPathTarget)
err = os.MkdirAll(bindParent, 0750)
if err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
}
t, err := os.Lstat(subpath.Path)
if err != nil {
return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindPathTarget>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
return false, bindPathTarget, nil
}
func getSubpathBindTarget(subpath Subpath) string {
// containerName is DNS label, i.e. safe as a directory name.
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
// Evaluate all symlinks here once for all subsequent functions.
newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
newPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
subpath.VolumePath = newVolumePath
subpath.Path = newPath
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
kubeletPid := os.Getpid()
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount
options := []string{"bind"}
klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
// This implementation is shared between Linux and NsEnterMounter
func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
containerDirs, err := ioutil.ReadDir(subPathDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("error reading %s: %s", subPathDir, err)
}
for _, containerDir := range containerDirs {
if !containerDir.IsDir() {
klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
continue
}
klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
if path == fullContainerDirPath {
// Skip top level directory
return nil
}
// pass through errors and let doCleanSubPath handle them
if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
return err
}
return nil
})
if err != nil {
return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
}
// Whole container has been processed, remove its directory.
if err := os.Remove(fullContainerDirPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
}
klog.V(5).Infof("Removed %s", fullContainerDirPath)
}
// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
if err := os.Remove(subPathDir); err != nil {
return fmt.Errorf("error deleting %s: %s", subPathDir, err)
}
klog.V(5).Infof("Removed %s", subPathDir)
// Remove entire subpath directory if it's the last one
podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
}
klog.V(5).Infof("Removed %s", podSubPathDir)
return nil
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
}
klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
return nil
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
return err
}
// Recusively remove directories if empty
if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
return err
}
return nil
}
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
s, err := os.Stat(curDir)
if err != nil {
if os.IsNotExist(err) {
klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
continue
}
return fmt.Errorf("error stat %q: %v", curDir, err)
}
if !s.IsDir() {
return fmt.Errorf("path %q not a directory", curDir)
}
err = os.Remove(curDir)
if os.IsExist(err) {
klog.V(5).Infof("Directory %q not empty, not removing", curDir)
break
} else if err != nil {
return fmt.Errorf("error removing directory %q: %v", curDir, err)
}
klog.V(5).Infof("Removed directory %q", curDir)
}
return nil
}
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(pathname); os.IsNotExist(err) {
return []string{}, nil
@ -1049,237 +764,6 @@ func getMode(pathname string) (os.FileMode, error) {
return info.Mode(), nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
parentFD, err := doSafeOpen(fullExistingPath, base)
if err != nil {
return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
}
childFD := -1
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
}
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink.
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
if err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
// Dive into the created directory
childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
if err != nil {
return fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// We can be sure that childFD is safe to use. It could be changed
// by user after Mkdirat() and before Openat(), however:
// - it could not be changed to symlink - we use nofollowFlags
// - it could be changed to a file (or device, pipe, socket, ...)
// but either subsequent Mkdirat() fails or we mount this file
// to user's container. Security is no violated in both cases
// and user either gets error or the file that it can already access.
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
parentFD = childFD
childFD = -1
}
// Everything was created. mkdirat(..., perm) above was affected by current
// umask and we must apply the right permissions to the last directory
// (that's the one that will be available to the container as subpath)
// so user can read/write it. This is the behavior of previous code.
// TODO: chmod all created directories, not just the last one.
// parentFD is the last created directory.
// Translate perm (os.FileMode) to uint32 that fchmod() expects
kernelPerm := uint32(perm & os.ModePerm)
if perm&os.ModeSetgid > 0 {
kernelPerm |= syscall.S_ISGID
}
if perm&os.ModeSetuid > 0 {
kernelPerm |= syscall.S_ISUID
}
if perm&os.ModeSticky > 0 {
kernelPerm |= syscall.S_ISVTX
}
if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
return fmt.Errorf("chmod %q failed: %s", currentPath, err)
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
dirs := strings.Split(rel, string(filepath.Separator))
// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
// This should be faster than looping through all dirs and calling os.Stat()
// on each of them, as the symlinks are resolved only once with OpenAt().
currentPath := base
fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
if err != nil {
return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
}
defer func() {
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
}()
for i, dir := range dirs {
// Using O_PATH here will prevent hangs in case user replaces directory with
// fifo
childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
if err != nil {
if os.IsNotExist(err) {
return currentPath, dirs[i:], nil
}
return base, nil, err
}
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
fd = childFD
currentPath = filepath.Join(currentPath, dir)
}
return pathname, []string{}, nil
}
// This implementation is shared between Linux and NsEnterMounter
// Open path and return its fd.
// Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) {
pathname = filepath.Clean(pathname)
base = filepath.Clean(base)
// Calculate segments to follow
subpath, err := filepath.Rel(base, pathname)
if err != nil {
return -1, err
}
segments := strings.Split(subpath, string(filepath.Separator))
// Assumption: base is the only directory that we have under control.
// Base dir is not allowed to be a symlink.
parentFD, err := syscall.Open(base, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
}
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
}
}
}()
childFD := -1
defer func() {
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := base
// Follow the segments one by one using openat() to make
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
klog.V(5).Infof("Opening path %s", currentPath)
childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
}
var deviceStat unix.Stat_t
err := unix.Fstat(childFD, &deviceStat)
if err != nil {
return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
}
fileFmt := deviceStat.Mode & syscall.S_IFMT
if fileFmt == syscall.S_IFLNK {
return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
}
// Close parentFD
if err = syscall.Close(parentFD); err != nil {
return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
}
// Set child to new parent
parentFD = childFD
childFD = -1
}
// We made it to the end, return this fd, don't close it
finalFD := parentFD
parentFD = -1
return finalFD, nil
}
// searchMountPoints finds all mount references to the source, returns a list of
// mountpoints.
// This function assumes source cannot be device.

1136
pkg/util/mount/mount_linux_test.go
View File

File diff suppressed because it is too large Load Diff

12
pkg/util/mount/mount_unsupported.go
View File

@ -110,18 +110,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return "", unsupportedErr
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, unsupportedErr
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return unsupportedErr
}
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return unsupportedErr
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

241
pkg/util/mount/mount_windows.go
View File

@ -26,7 +26,6 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"k8s.io/klog"
@ -279,123 +278,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return filepath.EvalSymlinks(pathname)
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(hostPath) == 0 {
return []uintptr{}, nil
}
finalSubPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
}
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
}
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
}
// lock all intermediate subPath directories and check they are all within volumePath
// volumePath & subPath should not contain any symlink, otherwise it will return error
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(subPath) == 0 {
return []uintptr{}, nil
}
// get relative path to volumePath
relSubPath, err := filepath.Rel(volumePath, subPath)
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if startsWithBackstep(relSubPath) {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
if relSubPath == "." {
// volumePath and subPath are equal
return []uintptr{}, nil
}
fileHandles := []uintptr{}
var errorResult error
currentFullPath := volumePath
dirs := strings.Split(relSubPath, string(os.PathSeparator))
for _, dir := range dirs {
// lock intermediate subPath directory first
currentFullPath = filepath.Join(currentFullPath, dir)
handle, err := lockPath(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
break
}
fileHandles = append(fileHandles, handle)
// make sure intermediate subPath directory does not contain symlink any more
stat, err := os.Lstat(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
break
}
if stat.Mode()&os.ModeSymlink != 0 {
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
break
}
if !PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
}
return fileHandles, errorResult
}
// unlockPath unlock directories
func unlockPath(fileHandles []uintptr) {
if fileHandles != nil {
for _, handle := range fileHandles {
syscall.CloseHandle(syscall.Handle(handle))
}
}
}
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
func lockPath(path string) (uintptr, error) {
if len(path) == 0 {
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return uintptr(syscall.InvalidHandle), err
}
access := uint32(syscall.GENERIC_READ)
sharemode := uint32(syscall.FILE_SHARE_READ)
createmode := uint32(syscall.OPEN_EXISTING)
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
return uintptr(fd), err
}
// Lock all directories in subPath and check they're not symlinks.
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
cleanupAction = func() {
unlockPath(handles)
}
return subPath.Path, cleanupAction, err
}
// No bind-mounts for subpaths are necessary on Windows
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
// Try to mount the disk
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
@ -522,126 +404,3 @@ func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
}
return info.Mode(), nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
if len(toCreate) == 0 {
return nil
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
}
fullBasePath, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
defer unlockPath(fileHandles)
if err != nil {
return err
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink by lock that directory immediately
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
if err := os.Mkdir(currentPath, perm); err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
handle, err := lockPath(currentPath)
if err != nil {
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
}
defer syscall.CloseHandle(syscall.Handle(handle))
// make sure newly created directory does not contain symlink after lock
stat, err := os.Lstat(currentPath)
if err != nil {
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
}
if stat.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
}
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
if startsWithBackstep(rel) {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}
if rel == "." {
// base and pathname are equal
return pathname, []string{}, nil
}
dirs := strings.Split(rel, string(filepath.Separator))
parent := base
currentPath := base
for i, dir := range dirs {
parent = currentPath
currentPath = filepath.Join(parent, dir)
if _, err := os.Lstat(currentPath); err != nil {
if os.IsNotExist(err) {
return parent, dirs[i:], nil
}
return base, nil, err
}
}
return pathname, []string{}, nil
}

403
pkg/util/mount/mount_windows_test.go
View File

@ -134,409 +134,6 @@ func TestGetMountRefs(t *testing.T) {
}
}
func TestDoSafeMakeDir(t *testing.T) {
base, err := ioutil.TempDir("", "TestDoSafeMakeDir")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
os.MkdirAll(testingVolumePath, 0755)
defer os.RemoveAll(testingVolumePath)
tests := []struct {
volumePath string
subPath string
expectError bool
symlinkTarget string
}{
{
volumePath: testingVolumePath,
subPath: ``,
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `x`),
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectError: false,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink\c\d`),
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink\y926`),
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\symlink`),
expectError: false,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\x\symlink`),
expectError: false,
symlinkTarget: filepath.Join(testingVolumePath, `a`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755))
if test.expectError {
assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
continue
}
assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
if _, err := os.Stat(test.subPath); os.IsNotExist(err) {
t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
}
}
}
func TestLockAndCheckSubPath(t *testing.T) {
base, err := ioutil.TempDir("", "TestLockAndCheckSubPath")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
tests := []struct {
volumePath string
subPath string
expectedHandleCount int
expectError bool
symlinkTarget string
}{
{
volumePath: `c:\`,
subPath: ``,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: ``,
subPath: `a`,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a`),
expectedHandleCount: 1,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectedHandleCount: 4,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectedHandleCount: 0,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`),
expectedHandleCount: 0,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
expectedHandleCount: 2,
expectError: false,
symlinkTarget: filepath.Join(testingVolumePath, `a\b`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 {
os.MkdirAll(test.volumePath, 0755)
if len(test.symlinkTarget) == 0 {
// make all intermediate sub directories
os.MkdirAll(test.subPath, 0755)
} else {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
}
fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath)
unlockPath(fileHandles)
assert.Equal(t, test.expectedHandleCount, len(fileHandles))
if test.expectError {
assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
continue
}
assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) {
base, err := ioutil.TempDir("", "TestLockAndCheckSubPathWithoutSymlink")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
tests := []struct {
volumePath string
subPath string
expectedHandleCount int
expectError bool
symlinkTarget string
}{
{
volumePath: `c:\`,
subPath: ``,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: ``,
subPath: `a`,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a`),
expectedHandleCount: 1,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectedHandleCount: 4,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectedHandleCount: 1,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`),
expectedHandleCount: 4,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
expectedHandleCount: 5,
expectError: true,
symlinkTarget: filepath.Join(testingVolumePath, `a\b`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 {
os.MkdirAll(test.volumePath, 0755)
if len(test.symlinkTarget) == 0 {
// make all intermediate sub directories
os.MkdirAll(test.subPath, 0755)
} else {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
}
fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath)
unlockPath(fileHandles)
assert.Equal(t, test.expectedHandleCount, len(fileHandles))
if test.expectError {
assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
continue
}
assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestFindExistingPrefix(t *testing.T) {
base, err := ioutil.TempDir("", "TestFindExistingPrefix")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
tests := []struct {
base string
pathname string
expectError bool
expectedExistingPath string
expectedToCreateDirs []string
createSubPathBeforeTest bool
}{
{
base: `c:\tmp\a`,
pathname: `c:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: ``,
pathname: `c:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: `c:\tmp\a`,
pathname: `d:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: testingVolumePath,
expectError: false,
expectedExistingPath: testingVolumePath,
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{},
createSubPathBeforeTest: true,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b\c\`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{`c`},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b\c\d`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{`c`, `d`},
createSubPathBeforeTest: false,
},
}
for _, test := range tests {
if test.createSubPathBeforeTest {
os.MkdirAll(test.pathname, 0755)
}
existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname)
if test.expectError {
assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname)
continue
}
assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname)
assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
test.base, test.pathname, existingPath, test.expectedExistingPath)
assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
test.base, test.pathname, toCreate, test.expectedToCreateDirs)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestPathWithinBase(t *testing.T) {
tests := []struct {
fullPath string

136
pkg/util/mount/nsenter_mount.go
View File

@ -23,9 +23,7 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/utils/nsenter"
utilpath "k8s.io/utils/path"
@ -298,51 +296,6 @@ func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error)
return mounter.ne.EvalSymlinks(pathname, true)
}
func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(mounter.rootDir)
if PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
exists, err := mounter.ExistsPath(pathname)
if err != nil {
@ -358,95 +311,6 @@ func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return searchMountPoints(hostpath, hostProcMountinfoPath)
}
func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {

294
pkg/util/mount/nsenter_mount_test.go
View File

@ -23,10 +23,8 @@ import (
"os"
"os/user"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
"k8s.io/utils/nsenter"
)
@ -76,80 +74,6 @@ func TestParseFindMnt(t *testing.T) {
}
}
func TestCheckDeviceInode(t *testing.T) {
testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
if err != nil {
t.Fatalf("Cannot create temporary directory: %s", err)
}
defer os.RemoveAll(testDir)
tests := []struct {
name string
srcPath string
dstPath string
expectError string
}{
{
name: "the same file",
srcPath: filepath.Join(testDir, "1"),
dstPath: filepath.Join(testDir, "1"),
expectError: "",
},
{
name: "different file on the same FS",
srcPath: filepath.Join(testDir, "2.1"),
dstPath: filepath.Join(testDir, "2.2"),
expectError: "different inode",
},
{
name: "different file on different device",
srcPath: filepath.Join(testDir, "3"),
// /proc is always on a different "device" than /tmp (or $TEMP)
dstPath: "/proc/self/status",
expectError: "different device",
},
}
for _, test := range tests {
if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil {
t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err)
continue
}
// Don't create dst if it exists
if _, err := os.Stat(test.dstPath); os.IsNotExist(err) {
if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil {
t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err)
continue
}
} else if err != nil {
t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err)
continue
}
fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644)
if err != nil {
t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err)
continue
}
err = checkDeviceInode(fd, test.dstPath)
if test.expectError == "" && err != nil {
t.Errorf("Test %q: expected no error, got %s", test.name, err)
}
if test.expectError != "" {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), test.expectError) {
t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err)
}
}
}
}
}
func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) {
rootfsPath = filepath.Join(tmpdir, "rootfs")
if err := os.Mkdir(rootfsPath, 0755); err != nil {
@ -504,221 +428,3 @@ func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) {
}
return fullPath, nil
}
func TestNsenterSafeMakeDir(t *testing.T) {
tests := []struct {
name string
prepare func(base, rootfs, varlib string) (expectedDir string, err error)
subdir string
expectError bool
// If true, "base" directory for SafeMakeDir will be /var/lib/kubelet
baseIsVarLib bool
}{
{
name: "simple directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /roots/
expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "simple existing directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: directory exists
hostPath := filepath.Join(base, "some/subdirectory/structure")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
// In rootfs: directory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected to be created in /roots/
expectedDir = kubeletPath
return expectedDir, nil
},
},
{
name: "absolute symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to /base/other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
otherPath := filepath.Join(base, "other")
if err := os.Symlink(otherPath, somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "relative symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "symlink into unsafe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/some is link to /bin/other
somePath := filepath.Join(base, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
{
name: "simple directory in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "safe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other
hostPath := filepath.Join(varlib, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "other/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "unsafe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/some is link to /bin
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
// Prepare base directory for the test
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
// Prepare base directory also in /rootfs
rootfsBase := filepath.Join(rootfs, testBase)
if err := os.MkdirAll(rootfsBase, 0755); err != nil {
t.Error(err)
continue
}
expectedDir := ""
if test.prepare != nil {
expectedDir, err = test.prepare(testBase, rootfs, varlib)
if err != nil {
t.Error(err)
continue
}
}
if test.baseIsVarLib {
// use /var/lib/kubelet as the test base so we can test creating
// subdirs there directly in /var/lib/kubenet and not in
// /rootfs/var/lib/kubelet
testBase = varlib
}
err = mounter.SafeMakeDir(test.subdir, testBase, 0755)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if test.expectError {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), "is outside of allowed base") {
t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err)
}
}
}
if expectedDir != "" {
_, err := os.Stat(expectedDir)
if err != nil {
t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err)
}
}
}
}

12
pkg/util/mount/nsenter_mount_unsupported.go
View File

@ -93,18 +93,6 @@ func (*NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

1
pkg/volume/BUILD
View File

@ -21,6 +21,7 @@ go_library(
"//pkg/util/mount:go_default_library",
"//pkg/volume/util/fs:go_default_library",
"//pkg/volume/util/recyclerclient:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",

4
pkg/volume/plugins.go
View File

@ -36,6 +36,7 @@ import (
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume/util/recyclerclient"
"k8s.io/kubernetes/pkg/volume/util/subpath"
)
type ProbeOperation uint32
@ -367,6 +368,9 @@ type VolumeHost interface {
// Returns the event recorder of kubelet.
GetEventRecorder() record.EventRecorder
// Returns an interface that should be used to execute subpath operations
GetSubpather() subpath.Interface
}
// VolumePluginMgr tracks registered plugins.

1
pkg/volume/testing/BUILD
View File

@ -17,6 +17,7 @@ go_library(
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/recyclerclient:go_default_library",
"//pkg/volume/util/subpath:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",

7
pkg/volume/testing/testing.go
View File

@ -42,6 +42,7 @@ import (
. "k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/recyclerclient"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
utilstrings "k8s.io/utils/strings"
)
@ -71,6 +72,7 @@ type fakeVolumeHost struct {
exec mount.Exec
nodeLabels map[string]string
nodeName string
subpather subpath.Interface
}
func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost {
@ -101,6 +103,7 @@ func newFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins [
}
host.exec = mount.NewFakeExec(nil)
host.pluginMgr.InitPlugins(plugins, nil /* prober */, host)
host.subpather = &subpath.FakeSubpath{}
return host
}
@ -149,6 +152,10 @@ func (f *fakeVolumeHost) GetMounter(pluginName string) mount.Interface {
return f.mounter
}
func (f *fakeVolumeHost) GetSubpather() subpath.Interface {
return f.subpather
}
func (f *fakeVolumeHost) NewWrapperMounter(volName string, spec Spec, pod *v1.Pod, opts VolumeOptions) (Mounter, error) {
// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
wrapperVolumeName := "wrapped_" + volName

1
pkg/volume/util/BUILD
View File

@ -88,6 +88,7 @@ filegroup(
"//pkg/volume/util/nestedpendingoperations:all-srcs",
"//pkg/volume/util/operationexecutor:all-srcs",
"//pkg/volume/util/recyclerclient:all-srcs",
"//pkg/volume/util/subpath:all-srcs",
"//pkg/volume/util/types:all-srcs",
"//pkg/volume/util/volumepathhandler:all-srcs",
],

4
pkg/volume/util/operationexecutor/operation_generator.go
View File

@ -750,11 +750,11 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc(
}
unmountVolumeFunc := func() (error, error) {
mounter := og.volumePluginMgr.Host.GetMounter(volumeToUnmount.PluginName)
subpather := og.volumePluginMgr.Host.GetSubpather()
// Remove all bind-mounts for subPaths
podDir := path.Join(podsDir, string(volumeToUnmount.PodUID))
if err := mounter.CleanSubPaths(podDir, volumeToUnmount.InnerVolumeSpecName); err != nil {
if err := subpather.CleanSubPaths(podDir, volumeToUnmount.InnerVolumeSpecName); err != nil {
return volumeToUnmount.GenerateError("error cleaning subPath mounts", err)
}

100
pkg/volume/util/subpath/BUILD Normal file
View File

@ -0,0 +1,100 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"subpath.go",
"subpath_linux.go",
"subpath_nsenter.go",
"subpath_unsupported.go",
"subpath_windows.go",
],
importpath = "k8s.io/kubernetes/pkg/volume/util/subpath",
visibility = ["//visibility:public"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/mount:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//pkg/util/mount:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"//conditions:default": [],
}),
)
go_test(
name = "go_default_test",
srcs = [
"subpath_linux_test.go",
"subpath_nsenter_test.go",
"subpath_windows_test.go",
],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/mount:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

13
pkg/volume/util/subpath/OWNERS Normal file
View File

@ -0,0 +1,13 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- jingxu97
- saad-ali
- jsafrane
- msau42
- andyzhangx
approvers:
- jingxu97
- saad-ali
- jsafrane

92
pkg/volume/util/subpath/subpath.go Normal file
View File

@ -0,0 +1,92 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import "os"
// Interface defines the set of methods all subpathers must implement
type Interface interface {
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(poodDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
// SafeMakeDir creates subdir within given base. It makes sure that the
// created directory does not escape given base directory mis-using
// symlinks. Note that the function makes sure that it creates the directory
// somewhere under the base, nothing else. E.g. if the directory already
// exists, it may exist outside of the base due to symlinks.
// This method should be used if the directory to create is inside volume
// that's under user control. User must not be able to use symlinks to
// escape the volume to create directories somewhere else.
SafeMakeDir(subdir string, base string, perm os.FileMode) error
}
// Subpath defines the attributes of a subpath
type Subpath struct {
// index of the VolumeMount for this container
VolumeMountIndex int
// Full path to the subpath directory on the host
Path string
// name of the volume that is a valid directory name.
VolumeName string
// Full path to the volume path
VolumePath string
// Path to the pod's directory, including pod UID
PodDir string
// Name of the container
ContainerName string
}
// Compile time-check for all implementers of subpath interface
var _ Interface = &subpath{}
var _ Interface = &FakeSubpath{}
// FakeSubpath is a subpather implementation for testing
type FakeSubpath struct{}
// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns
// newHostPath == subPath.Path
func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop
func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop
func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}

838
pkg/volume/util/subpath/subpath_linux.go
View File

@ -16,44 +16,25 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
package subpath
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strconv"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
utilexec "k8s.io/utils/exec"
utilio "k8s.io/utils/io"
utilpath "k8s.io/utils/path"
"k8s.io/kubernetes/pkg/util/mount"
)
const (
// How many times to retry for a consistent read of /proc/mounts.
maxListTries = 3
// Number of fields per line in /proc/mounts as per the fstab man page.
expectedNumFieldsPerLine = 6
// At least number of fields per line in /proc/<pid>/mountinfo.
expectedAtLeastNumFieldsPerMountInfo = 10
// Location of the mount file to use
procMountsPath = "/proc/mounts"
// Location of the mountinfo file
procMountInfoPath = "/proc/self/mountinfo"
// 'fsck' found errors and corrected them
fsckErrorsCorrected = 1
// 'fsck' found errors but exited without correcting them
fsckErrorsUncorrected = 4
// place for subpath mounts
// TODO: pass in directory using kubelet_getters instead
containerSubPathDirectoryName = "volume-subpaths"
@ -63,671 +44,34 @@ const (
openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
)
// Mounter provides the default implementation of mount.Interface
// for the linux platform. This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct {
mounterPath string
withSystemd bool
type subpath struct {
mounter mount.Interface
}
// New returns a mount.Interface for the current system.
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
func New(mounterPath string) Interface {
return &Mounter{
mounterPath: mounterPath,
withSystemd: detectSystemd(),
// New returns a subpath.Interface for the current system
func New(mounter mount.Interface) Interface {
return &subpath{
mounter: mounter,
}
}
// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must
// be an empty string in case it's not required, e.g. for remount, or for auto filesystem
// type, where kernel handles fstype for you. The mount 'options' is a list of options,
// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
// required, call Mount with an empty string list or nil.
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
// Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
// All Linux distros are expected to be shipped with a mount utility that a support bind mounts.
mounterPath := ""
bind, bindOpts, bindRemountOpts := isBind(options)
if bind {
err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts)
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(sp.mounter, podDir, volumeName)
}
func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return err
}
return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts)
}
// The list of filesystems that require containerized mounter on GCI image cluster
fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs")
if fsTypesNeedMounter.Has(fstype) {
mounterPath = mounter.mounterPath
}
return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options)
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error {
mountArgs := makeMountArgs(source, target, fstype, options)
if len(mounterPath) > 0 {
mountArgs = append([]string{mountCmd}, mountArgs...)
mountCmd = mounterPath
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
if m.withSystemd {
// Try to run mount via systemd-run --scope. This will escape the
// service where kubelet runs and any fuse daemons will be started in a
// specific scope. kubelet service than can be restarted without killing
// these fuse daemons.
//
// Complete command line (when mounterPath is not used):
// systemd-run --description=... --scope -- mount -t <type> <what> <where>
//
// Expected flow:
// * systemd-run creates a transient scope (=~ cgroup) and executes its
// argument (/bin/mount) there.
// * mount does its job, forks a fuse daemon if necessary and finishes.
// (systemd-run --scope finishes at this point, returning mount's exit
// code and stdout/stderr - thats one of --scope benefits).
// * systemd keeps the fuse daemon running in the scope (i.e. in its own
// cgroup) until the fuse daemon dies (another --scope benefit).
// Kubelet service can be restarted and the fuse daemon survives.
// * When the fuse daemon dies (e.g. during unmount) systemd removes the
// scope automatically.
//
// systemd-mount is not used because it's too new for older distros
// (CentOS 7, Debian Jessie).
mountCmd, mountArgs = addSystemdScope("systemd-run", target, mountCmd, mountArgs)
} else {
// No systemd-run on the host (or we failed to check it), assume kubelet
// does not run as a systemd service.
// No code here, mountCmd and mountArgs are already populated.
}
klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs)
command := exec.Command(mountCmd, mountArgs...)
output, err := command.CombinedOutput()
if err != nil {
args := strings.Join(mountArgs, " ")
klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output))
return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n",
err, mountCmd, args, string(output))
}
return err
}
// detectSystemd returns true if OS runs with systemd as init. When not sure
// (permission errors, ...), it returns false.
// There may be different ways how to detect systemd, this one makes sure that
// systemd-runs (needed by Mount()) works.
func detectSystemd() bool {
if _, err := exec.LookPath("systemd-run"); err != nil {
klog.V(2).Infof("Detected OS without systemd")
return false
}
// Try to run systemd-run --scope /bin/true, that should be enough
// to make sure that systemd is really running and not just installed,
// which happens when running in a container with a systemd-based image
// but with different pid 1.
cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true")
output, err := cmd.CombinedOutput()
if err != nil {
klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS")
klog.V(4).Infof("systemd-run failed with: %v", err)
klog.V(4).Infof("systemd-run output: %s", string(output))
return false
}
klog.V(2).Infof("Detected OS with systemd")
return true
}
// makeMountArgs makes the arguments to the mount(8) command.
func makeMountArgs(source, target, fstype string, options []string) []string {
// Build mount command as follows:
// mount [-t $fstype] [-o $options] [$source] $target
mountArgs := []string{}
if len(fstype) > 0 {
mountArgs = append(mountArgs, "-t", fstype)
}
if len(options) > 0 {
mountArgs = append(mountArgs, "-o", strings.Join(options, ","))
}
if len(source) > 0 {
mountArgs = append(mountArgs, source)
}
mountArgs = append(mountArgs, target)
return mountArgs
}
// addSystemdScope adds "system-run --scope" to given command line
func addSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) {
descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName)
systemdRunArgs := []string{descriptionArg, "--scope", "--", command}
return systemdRunPath, append(systemdRunArgs, args...)
}
// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
klog.V(4).Infof("Unmounting %s", target)
command := exec.Command("umount", target)
output, err := command.CombinedOutput()
if err != nil {
return fmt.Errorf("Unmount failed: %v\nUnmounting arguments: %s\nOutput: %s\n", err, target, string(output))
}
return nil
}
// List returns a list of all mounted filesystems.
func (*Mounter) List() ([]MountPoint, error) {
return listProcMounts(procMountsPath)
}
func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
deletedDir := fmt.Sprintf("%s\\040(deleted)", dir)
return ((mp.Path == dir) || (mp.Path == deletedDir))
}
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return isNotMountPoint(mounter, dir)
}
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
// It is fast but not necessarily ALWAYS correct. If the path is in fact
// a bind mount from one part of a mount to another it will not be detected.
// mkdir /tmp/a /tmp/b; mount --bin /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b")
// will return true. When in fact /tmp/b is a mount point. If this situation
// if of interest to you, don't use this function...
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
stat, err := os.Stat(file)
if err != nil {
return true, err
}
rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/")))
if err != nil {
return true, err
}
// If the directory has a different device as parent, then it is a mountpoint.
if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev {
return false, nil
}
return true, nil
}
// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// If pathname is not a device, log and return false with nil error.
// If open returns errno EBUSY, return true with nil error.
// If open returns nil, return false with nil error.
// Otherwise, return false with error
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
return exclusiveOpenFailsOnDevice(pathname)
}
// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
// to a device.
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
pathType, err := mounter.GetFileType(pathname)
isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev
return isDevice, err
}
func exclusiveOpenFailsOnDevice(pathname string) (bool, error) {
var isDevice bool
finfo, err := os.Stat(pathname)
if os.IsNotExist(err) {
isDevice = false
}
// err in call to os.Stat
if err != nil {
return false, fmt.Errorf(
"PathIsDevice failed for path %q: %v",
pathname,
err)
}
// path refers to a device
if finfo.Mode()&os.ModeDevice != 0 {
isDevice = true
}
if !isDevice {
klog.Errorf("Path %q is not referring to a device.", pathname)
return false, nil
}
fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL, 0)
// If the device is in use, open will return an invalid fd.
// When this happens, it is expected that Close will fail and throw an error.
defer unix.Close(fd)
if errno == nil {
// device not in use
return false, nil
} else if errno == unix.EBUSY {
// device is in use
return true, nil
}
// error during call to Open
return false, errno
}
//GetDeviceNameFromMount: given a mount point, find the device name from its global mount point
func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
return getDeviceNameFromMount(mounter, mountPath, pluginDir)
}
// getDeviceNameFromMount find the device name from /proc/mounts in which
// the mount path reference should match the given plugin directory. In case no mount path reference
// matches, returns the volume name taken from its given mountPath
func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) {
refs, err := mounter.GetMountRefs(mountPath)
if err != nil {
klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
return "", err
}
if len(refs) == 0 {
klog.V(4).Infof("Directory %s is not mounted", mountPath)
return "", fmt.Errorf("directory %s is not mounted", mountPath)
}
basemountPath := path.Join(pluginDir, MountsInGlobalPDPath)
for _, ref := range refs {
if strings.HasPrefix(ref, basemountPath) {
volumeID, err := filepath.Rel(basemountPath, ref)
if err != nil {
klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
return "", err
}
return volumeID, nil
}
}
return path.Base(mountPath), nil
}
func listProcMounts(mountFilePath string) ([]MountPoint, error) {
content, err := utilio.ConsistentRead(mountFilePath, maxListTries)
if err != nil {
return nil, err
}
return parseProcMounts(content)
}
func parseProcMounts(content []byte) ([]MountPoint, error) {
out := []MountPoint{}
lines := strings.Split(string(content), "\n")
for _, line := range lines {
if line == "" {
// the last split() item is empty string following the last \n
continue
}
fields := strings.Fields(line)
if len(fields) != expectedNumFieldsPerLine {
return nil, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line)
}
mp := MountPoint{
Device: fields[0],
Path: fields[1],
Type: fields[2],
Opts: strings.Split(fields[3], ","),
}
freq, err := strconv.Atoi(fields[4])
if err != nil {
return nil, err
}
mp.Freq = freq
pass, err := strconv.Atoi(fields[5])
if err != nil {
return nil, err
}
mp.Pass = pass
out = append(out, mp)
}
return out, nil
}
func (mounter *Mounter) MakeRShared(path string) error {
return doMakeRShared(path, procMountInfoPath)
}
func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
return getFileType(pathname)
}
func (mounter *Mounter) MakeDir(pathname string) error {
err := os.MkdirAll(pathname, os.FileMode(0755))
if err != nil {
if !os.IsExist(err) {
return err
}
}
return nil
}
func (mounter *Mounter) MakeFile(pathname string) error {
f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644))
defer f.Close()
if err != nil {
if !os.IsExist(err) {
return err
}
}
return nil
}
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}
func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return filepath.EvalSymlinks(pathname)
}
// formatAndMount uses unix utils to format and mount the given disk
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
readOnly := false
for _, option := range options {
if option == "ro" {
readOnly = true
break
}
}
options = append(options, "defaults")
if !readOnly {
// Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw.
klog.V(4).Infof("Checking for issues with fsck on disk: %s", source)
args := []string{"-a", source}
out, err := mounter.Exec.Run("fsck", args...)
if err != nil {
ee, isExitError := err.(utilexec.ExitError)
switch {
case err == utilexec.ErrExecutableNotFound:
klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.")
case isExitError && ee.ExitStatus() == fsckErrorsCorrected:
klog.Infof("Device %s has errors which were corrected by fsck.", source)
case isExitError && ee.ExitStatus() == fsckErrorsUncorrected:
return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s.", source, string(out))
case isExitError && ee.ExitStatus() > fsckErrorsUncorrected:
klog.Infof("`fsck` error %s", string(out))
}
}
}
// Try to mount the disk
klog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target)
mountErr := mounter.Interface.Mount(source, target, fstype, options)
if mountErr != nil {
// Mount failed. This indicates either that the disk is unformatted or
// it contains an unexpected filesystem.
existingFormat, err := mounter.GetDiskFormat(source)
if err != nil {
return err
}
if existingFormat == "" {
if readOnly {
// Don't attempt to format if mounting as readonly, return an error to reflect this.
return errors.New("failed to mount unformatted volume as read only")
}
// Disk is unformatted so format it.
args := []string{source}
// Use 'ext4' as the default
if len(fstype) == 0 {
fstype = "ext4"
}
if fstype == "ext4" || fstype == "ext3" {
args = []string{
"-F", // Force flag
"-m0", // Zero blocks reserved for super-user
source,
}
}
klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args)
_, err := mounter.Exec.Run("mkfs."+fstype, args...)
if err == nil {
// the disk has been formatted successfully try to mount it again.
klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target)
return mounter.Interface.Mount(source, target, fstype, options)
}
klog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err)
return err
} else {
// Disk is already formatted and failed to mount
if len(fstype) == 0 || fstype == existingFormat {
// This is mount error
return mountErr
} else {
// Block device is formatted with unexpected filesystem, let the user know
return fmt.Errorf("failed to mount the volume as %q, it already contains %s. Mount error: %v", fstype, existingFormat, mountErr)
}
}
}
return mountErr
}
// GetDiskFormat uses 'blkid' to see if the given disk is unformated
func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) {
args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args)
dataOut, err := mounter.Exec.Run("blkid", args...)
output := string(dataOut)
klog.V(4).Infof("Output: %q, err: %v", output, err)
if err != nil {
if exit, ok := err.(utilexec.ExitError); ok {
if exit.ExitStatus() == 2 {
// Disk device is unformatted.
// For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
// not found, or no (specified) devices could be identified, an
// exit code of 2 is returned.
return "", nil
}
}
klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err)
return "", err
}
var fstype, pttype string
lines := strings.Split(output, "\n")
for _, l := range lines {
if len(l) <= 0 {
// Ignore empty line.
continue
}
cs := strings.Split(l, "=")
if len(cs) != 2 {
return "", fmt.Errorf("blkid returns invalid output: %s", output)
}
// TYPE is filesystem type, and PTTYPE is partition table type, according
// to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/.
if cs[0] == "TYPE" {
fstype = cs[1]
} else if cs[0] == "PTTYPE" {
pttype = cs[1]
}
}
if len(pttype) > 0 {
klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype)
// Returns a special non-empty string as filesystem type, then kubelet
// will not format it.
return "unknown data, probably partitions", nil
}
return fstype, nil
}
// isShared returns true, if given path is on a mount point that has shared
// mount propagation.
func isShared(mount string, mountInfoPath string) (bool, error) {
info, err := findMountInfo(mount, mountInfoPath)
if err != nil {
return false, err
}
// parse optional parameters
for _, opt := range info.optionalFields {
if strings.HasPrefix(opt, "shared:") {
return true, nil
}
}
return false, nil
}
// This represents a single line in /proc/<pid>/mountinfo.
type mountInfo struct {
// Unique ID for the mount (maybe reused after umount).
id int
// The ID of the parent mount (or of self for the root of this mount namespace's mount tree).
parentID int
// The value of `st_dev` for files on this filesystem.
majorMinor string
// The pathname of the directory in the filesystem which forms the root of this mount.
root string
// Mount source, filesystem-specific information. e.g. device, tmpfs name.
source string
// Mount point, the pathname of the mount point.
mountPoint string
// Optional fieds, zero or more fields of the form "tag[:value]".
optionalFields []string
// The filesystem type in the form "type[.subtype]".
fsType string
// Per-mount options.
mountOptions []string
// Per-superblock options.
superOptions []string
}
// parseMountInfo parses /proc/xxx/mountinfo.
func parseMountInfo(filename string) ([]mountInfo, error) {
content, err := utilio.ConsistentRead(filename, maxListTries)
if err != nil {
return []mountInfo{}, err
}
contentStr := string(content)
infos := []mountInfo{}
for _, line := range strings.Split(contentStr, "\n") {
if line == "" {
// the last split() item is empty string following the last \n
continue
}
// See `man proc` for authoritative description of format of the file.
fields := strings.Fields(line)
if len(fields) < expectedAtLeastNumFieldsPerMountInfo {
return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line)
}
id, err := strconv.Atoi(fields[0])
if err != nil {
return nil, err
}
parentID, err := strconv.Atoi(fields[1])
if err != nil {
return nil, err
}
info := mountInfo{
id: id,
parentID: parentID,
majorMinor: fields[2],
root: fields[3],
mountPoint: fields[4],
mountOptions: strings.Split(fields[5], ","),
}
// All fields until "-" are "optional fields".
i := 6
for ; i < len(fields) && fields[i] != "-"; i++ {
info.optionalFields = append(info.optionalFields, fields[i])
}
// Parse the rest 3 fields.
i += 1
if len(fields)-i < 3 {
return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i)
}
info.fsType = fields[i]
info.source = fields[i+1]
info.superOptions = strings.Split(fields[i+2], ",")
infos = append(infos, info)
}
return infos, nil
}
func findMountInfo(path, mountInfoPath string) (mountInfo, error) {
infos, err := parseMountInfo(mountInfoPath)
if err != nil {
return mountInfo{}, err
}
// process /proc/xxx/mountinfo in backward order and find the first mount
// point that is prefix of 'path' - that's the mount where path resides
var info *mountInfo
for i := len(infos) - 1; i >= 0; i-- {
if PathWithinBase(path, infos[i].mountPoint) {
info = &infos[i]
break
}
}
if info == nil {
return mountInfo{}, fmt.Errorf("cannot find mount point for %q", path)
}
return *info, nil
}
// doMakeRShared is common implementation of MakeRShared on Linux. It checks if
// path is shared and bind-mounts it as rshared if needed. mountCmd and
// mountArgs are expected to contain mount-like command, doMakeRShared will add
// '--bind <path> <path>' and '--make-rshared <path>' to mountArgs.
func doMakeRShared(path string, mountInfoFilename string) error {
shared, err := isShared(path, mountInfoFilename)
if err != nil {
return err
}
if shared {
klog.V(4).Infof("Directory %s is already on a shared mount", path)
return nil
}
klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path)
// mount --bind /var/lib/kubelet /var/lib/kubelet
if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil {
return fmt.Errorf("failed to bind-mount %s: %v", path, err)
}
// mount --make-rshared /var/lib/kubelet
if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil {
return fmt.Errorf("failed to make %s rshared: %v", path, err)
}
return nil
}
// getSELinuxSupport is common implementation of GetSELinuxSupport on Linux.
func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
info, err := findMountInfo(path, mountInfoFilename)
if err != nil {
return false, err
}
// "seclabel" can be both in mount options and super options.
for _, opt := range info.superOptions {
if opt == "seclabel" {
return true, nil
}
}
for _, opt := range info.mountOptions {
if opt == "seclabel" {
return true, nil
}
}
return false, nil
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(mounter, subPath)
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(sp.mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
@ -735,9 +79,9 @@ func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string,
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnterMounter
func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
if !PathWithinBase(subpath.Path, subpath.VolumePath) {
// This implementation is shared between Linux and NsEnter
func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) {
if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
}
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
@ -754,7 +98,7 @@ func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := mounter.IsNotMountPoint(bindPathTarget)
@ -805,7 +149,7 @@ func getSubpathBindTarget(subpath Subpath) string {
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
@ -864,12 +208,8 @@ func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err err
return bindPathTarget, nil
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
// This implementation is shared between Linux and NsEnterMounter
func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
// This implementation is shared between Linux and NsEnter
func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error {
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
@ -929,12 +269,12 @@ func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil {
return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
}
@ -943,7 +283,7 @@ func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter Interface, subpath Subpath) error {
func cleanSubPath(mounter mount.Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
@ -962,7 +302,7 @@ func cleanSubPath(mounter Interface, subpath Subpath) error {
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !PathWithinBase(endDir, baseDir) {
if !mount.PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
@ -991,71 +331,13 @@ func removeEmptyDirs(baseDir, endDir string) error {
return nil
}
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(pathname); os.IsNotExist(err) {
return []string{}, nil
} else if err != nil {
return nil, err
}
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {
return nil, err
}
return searchMountPoints(realpath, procMountInfoPath)
}
func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
return getSELinuxSupport(pathname, procMountInfoPath)
}
func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {
return 0, err
}
return getFSGroup(realpath)
}
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
return getMode(pathname)
}
// This implementation is shared between Linux and NsEnterMounter
func getFSGroup(pathname string) (int64, error) {
info, err := os.Stat(pathname)
if err != nil {
return 0, err
}
return int64(info.Sys().(*syscall.Stat_t).Gid), nil
}
// This implementation is shared between Linux and NsEnterMounter
func getMode(pathname string) (os.FileMode, error) {
info, err := os.Stat(pathname)
if err != nil {
return 0, err
}
return info.Mode(), nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
if !mount.PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
@ -1082,7 +364,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !PathWithinBase(fullExistingPath, base) {
if !mount.PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
@ -1244,7 +526,7 @@ func doSafeOpen(pathname string, base string) (int, error) {
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !PathWithinBase(currentPath, base) {
if !mount.PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
@ -1279,51 +561,3 @@ func doSafeOpen(pathname string, base string) (int, error) {
return finalFD, nil
}
// searchMountPoints finds all mount references to the source, returns a list of
// mountpoints.
// This function assumes source cannot be device.
// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting,
// it's possible to mount a non-root path of a filesystem, so we need to use
// root path and major:minor to represent mount source uniquely.
// This implementation is shared between Linux and NsEnterMounter
func searchMountPoints(hostSource, mountInfoPath string) ([]string, error) {
mis, err := parseMountInfo(mountInfoPath)
if err != nil {
return nil, err
}
mountID := 0
rootPath := ""
majorMinor := ""
// Finding the underlying root path and major:minor if possible.
// We need search in backward order because it's possible for later mounts
// to overlap earlier mounts.
for i := len(mis) - 1; i >= 0; i-- {
if hostSource == mis[i].mountPoint || PathWithinBase(hostSource, mis[i].mountPoint) {
// If it's a mount point or path under a mount point.
mountID = mis[i].id
rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint))
majorMinor = mis[i].majorMinor
break
}
}
if rootPath == "" || majorMinor == "" {
return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource)
}
var refs []string
for i := range mis {
if mis[i].id == mountID {
// Ignore mount entry for mount source itself.
continue
}
if mis[i].root == rootPath && mis[i].majorMinor == majorMinor {
refs = append(refs, mis[i].mountPoint)
}
}
return refs, nil
}

912
pkg/volume/util/subpath/subpath_linux_test.go
View File

File diff suppressed because it is too large Load Diff

354
pkg/volume/util/subpath/subpath_nsenter.go
View File

@ -16,295 +16,48 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
package subpath
import (
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/utils/nsenter"
utilpath "k8s.io/utils/path"
"k8s.io/kubernetes/pkg/util/mount"
)
const (
// hostProcMountsPath is the default mount path for rootfs
hostProcMountsPath = "/rootfs/proc/1/mounts"
// hostProcMountinfoPath is the default mount info path for rootfs
hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
)
// Currently, all docker containers receive their own mount namespaces.
// NsenterMounter works by executing nsenter to run commands in
// the host's mount namespace.
type NsenterMounter struct {
type subpathNSE struct {
mounter mount.Interface
ne *nsenter.Nsenter
// rootDir is location of /var/lib/kubelet directory.
rootDir string
}
// NewNsenterMounter creates a new mounter for kubelet that runs as a container.
func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
return &NsenterMounter{
rootDir: rootDir,
// Compile time-check for all implementers of subpath interface
var _ Interface = &subpathNSE{}
// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
// It is only valid on Linux systems
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return &subpathNSE{
mounter: mounter,
ne: ne,
rootDir: rootDir,
}
}
// NsenterMounter implements mount.Interface
var _ = Interface(&NsenterMounter{})
// Mount runs mount(8) in the host's root mount namespace. Aside from this
// aspect, Mount has the same semantics as the mounter returned by mount.New()
func (n *NsenterMounter) Mount(source string, target string, fstype string, options []string) error {
bind, bindOpts, bindRemountOpts := isBind(options)
if bind {
err := n.doNsenterMount(source, target, fstype, bindOpts)
if err != nil {
return err
}
return n.doNsenterMount(source, target, fstype, bindRemountOpts)
func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(sp.mounter, podDir, volumeName)
}
return n.doNsenterMount(source, target, fstype, options)
}
// doNsenterMount nsenters the host's mount namespace and performs the
// requested mount.
func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error {
klog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options)
cmd, args := n.makeNsenterArgs(source, target, fstype, options)
outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput()
if len(outputBytes) != 0 {
klog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes))
}
return err
}
// makeNsenterArgs makes a list of argument to nsenter in order to do the
// requested mount.
func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) (string, []string) {
mountCmd := n.ne.AbsHostPath("mount")
mountArgs := makeMountArgs(source, target, fstype, options)
if systemdRunPath, hasSystemd := n.ne.SupportsSystemd(); hasSystemd {
// Complete command line:
// nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/systemd-run --description=... --scope -- /bin/mount -t <type> <what> <where>
// Expected flow is:
// * nsenter breaks out of container's mount namespace and executes
// host's systemd-run.
// * systemd-run creates a transient scope (=~ cgroup) and executes its
// argument (/bin/mount) there.
// * mount does its job, forks a fuse daemon if necessary and finishes.
// (systemd-run --scope finishes at this point, returning mount's exit
// code and stdout/stderr - thats one of --scope benefits).
// * systemd keeps the fuse daemon running in the scope (i.e. in its own
// cgroup) until the fuse daemon dies (another --scope benefit).
// Kubelet container can be restarted and the fuse daemon survives.
// * When the daemon dies (e.g. during unmount) systemd removes the
// scope automatically.
mountCmd, mountArgs = addSystemdScope(systemdRunPath, target, mountCmd, mountArgs)
} else {
// Fall back to simple mount when the host has no systemd.
// Complete command line:
// nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/mount -t <type> <what> <where>
// Expected flow is:
// * nsenter breaks out of container's mount namespace and executes host's /bin/mount.
// * mount does its job, forks a fuse daemon if necessary and finishes.
// * Any fuse daemon runs in cgroup of kubelet docker container,
// restart of kubelet container will kill it!
// No code here, mountCmd and mountArgs use /bin/mount
}
return mountCmd, mountArgs
}
// Unmount runs umount(8) in the host's mount namespace.
func (n *NsenterMounter) Unmount(target string) error {
args := []string{target}
// No need to execute systemd-run here, it's enough that unmount is executed
// in the host's mount namespace. It will finish appropriate fuse daemon(s)
// running in any scope.
klog.V(5).Infof("nsenter unmount args: %v", args)
outputBytes, err := n.ne.Exec("umount", args).CombinedOutput()
if len(outputBytes) != 0 {
klog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes))
}
return err
}
// List returns a list of all mounted filesystems in the host's mount namespace.
func (*NsenterMounter) List() ([]MountPoint, error) {
return listProcMounts(hostProcMountsPath)
}
func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
return isNotMountPoint(m, dir)
}
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
deletedDir := fmt.Sprintf("%s\\040(deleted)", dir)
return (mp.Path == dir) || (mp.Path == deletedDir)
}
// IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt
// in the host's root mount namespace.
func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) {
file, err := filepath.Abs(file)
if err != nil {
return true, err
}
// Check the directory exists
if _, err = os.Stat(file); os.IsNotExist(err) {
klog.V(5).Infof("findmnt: directory %s does not exist", file)
return true, err
}
// Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts
resolvedFile, err := n.EvalHostSymlinks(file)
if err != nil {
return true, err
}
// Add --first-only option: since we are testing for the absence of a mountpoint, it is sufficient to get only
// the first of multiple possible mountpoints using --first-only.
// Also add fstype output to make sure that the output of target file will give the full path
// TODO: Need more refactoring for this function. Track the solution with issue #26996
args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", resolvedFile}
klog.V(5).Infof("nsenter findmnt args: %v", args)
out, err := n.ne.Exec("findmnt", args).CombinedOutput()
if err != nil {
klog.V(2).Infof("Failed findmnt command for path %s: %s %v", resolvedFile, out, err)
// Different operating systems behave differently for paths which are not mount points.
// On older versions (e.g. 2.20.1) we'd get error, on newer ones (e.g. 2.26.2) we'd get "/".
// It's safer to assume that it's not a mount point.
return true, nil
}
mountTarget, err := parseFindMnt(string(out))
if err != nil {
return false, err
}
klog.V(5).Infof("IsLikelyNotMountPoint findmnt output for path %s: %v:", resolvedFile, mountTarget)
if mountTarget == resolvedFile {
klog.V(5).Infof("IsLikelyNotMountPoint: %s is a mount point", resolvedFile)
return false, nil
}
klog.V(5).Infof("IsLikelyNotMountPoint: %s is not a mount point", resolvedFile)
return true, nil
}
// parse output of "findmnt -o target,fstype" and return just the target
func parseFindMnt(out string) (string, error) {
// cut trailing newline
out = strings.TrimSuffix(out, "\n")
// cut everything after the last space - it's the filesystem type
i := strings.LastIndex(out, " ")
if i == -1 {
return "", fmt.Errorf("error parsing findmnt output, expected at least one space: %q", out)
}
return out[:i], nil
}
// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// Returns true if open returns errno EBUSY, and false if errno is nil.
// Returns an error if errno is any error other than EBUSY.
// Returns with error if pathname is not a device.
func (n *NsenterMounter) DeviceOpened(pathname string) (bool, error) {
return exclusiveOpenFailsOnDevice(pathname)
}
// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
// to a device.
func (n *NsenterMounter) PathIsDevice(pathname string) (bool, error) {
pathType, err := n.GetFileType(pathname)
isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev
return isDevice, err
}
//GetDeviceNameFromMount given a mount point, find the volume id from checking /proc/mounts
func (n *NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
return getDeviceNameFromMount(n, mountPath, pluginDir)
}
func (n *NsenterMounter) MakeRShared(path string) error {
return doMakeRShared(path, hostProcMountinfoPath)
}
func (mounter *NsenterMounter) GetFileType(pathname string) (FileType, error) {
var pathType FileType
outputBytes, err := mounter.ne.Exec("stat", []string{"-L", "--printf=%F", pathname}).CombinedOutput()
if err != nil {
if strings.Contains(string(outputBytes), "No such file") {
err = fmt.Errorf("%s does not exist", pathname)
} else {
err = fmt.Errorf("stat %s error: %v", pathname, string(outputBytes))
}
return pathType, err
}
switch string(outputBytes) {
case "socket":
return FileTypeSocket, nil
case "character special file":
return FileTypeCharDev, nil
case "block special file":
return FileTypeBlockDev, nil
case "directory":
return FileTypeDirectory, nil
case "regular file":
return FileTypeFile, nil
}
return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
}
func (mounter *NsenterMounter) MakeDir(pathname string) error {
args := []string{"-p", pathname}
if _, err := mounter.ne.Exec("mkdir", args).CombinedOutput(); err != nil {
return err
}
return nil
}
func (mounter *NsenterMounter) MakeFile(pathname string) error {
args := []string{pathname}
if _, err := mounter.ne.Exec("touch", args).CombinedOutput(); err != nil {
return err
}
return nil
}
func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) {
// Resolve the symlinks but allow the target not to exist. EvalSymlinks
// would return an generic error when the target does not exist.
hostPath, err := mounter.ne.EvalSymlinks(pathname, false /* mustExist */)
if err != nil {
return false, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return utilpath.Exists(utilpath.CheckFollowSymlink, kubeletpath)
}
func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return mounter.ne.EvalSymlinks(pathname, true)
}
func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
newHostPath, err = sp.doNsEnterBindSubPath(subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
@ -312,22 +65,22 @@ func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath
return newHostPath, cleanupAction, err
}
func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(mounter.rootDir)
if PathWithinBase(evaluatedBase, rootDir) {
rootDir := filepath.Clean(sp.rootDir)
if mount.PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
@ -338,27 +91,12 @@ func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.F
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := sp.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
exists, err := mounter.ExistsPath(pathname)
if err != nil {
return nil, err
}
if !exists {
return []string{}, nil
}
hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return nil, err
}
return searchMountPoints(hostpath, hostProcMountinfoPath)
}
func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
@ -367,26 +105,26 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(mounter, subpath)
fd, err := safeOpenSubPath(sp.mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
if err != nil {
return "", err
}
@ -399,7 +137,7 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
@ -409,7 +147,7 @@ func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath st
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
@ -446,25 +184,3 @@ func checkDeviceInode(fd int, path string) error {
}
return nil
}
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return -1, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return getFSGroup(kubeletpath)
}
func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
return getSELinuxSupport(pathname, hostProcMountsPath)
}
func (mounter *NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return 0, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return getMode(kubeletpath)
}

412
pkg/volume/util/subpath/subpath_nsenter_test.go
View File

@ -16,66 +16,22 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
package subpath
import (
"io/ioutil"
"os"
"os/user"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
"k8s.io/utils/nsenter"
"k8s.io/kubernetes/pkg/util/mount"
)
func TestParseFindMnt(t *testing.T) {
tests := []struct {
input string
target string
expectError bool
}{
{
// standard mount name, e.g. for AWS
"/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389 ext4\n",
"/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389",
false,
},
{
// mount name with space, e.g. vSphere
"/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n",
"/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk",
false,
},
{
// hypotetic mount with several spaces
"/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n",
"/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk",
false,
},
{
// invalid output - no filesystem type
"/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/blabla",
"",
true,
},
}
for i, test := range tests {
target, err := parseFindMnt(test.input)
if test.expectError && err == nil {
t.Errorf("test %d expected error, got nil", i)
}
if !test.expectError && err != nil {
t.Errorf("test %d returned error: %s", i, err)
}
if target != test.target {
t.Errorf("test %d expected %q, got %q", i, test.target, target)
}
}
}
func TestCheckDeviceInode(t *testing.T) {
testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
if err != nil {
@ -150,359 +106,22 @@ func TestCheckDeviceInode(t *testing.T) {
}
}
func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) {
rootfsPath = filepath.Join(tmpdir, "rootfs")
func newFakeNsenterMounter(tmpdir string, t *testing.T) (*mount.NsenterMounter, string, string, *nsenter.Nsenter, error) {
rootfsPath := filepath.Join(tmpdir, "rootfs")
if err := os.Mkdir(rootfsPath, 0755); err != nil {
return nil, "", "", err
return nil, "", "", nil, err
}
ne, err := nsenter.NewFakeNsenter(rootfsPath)
if err != nil {
return nil, "", "", err
return nil, "", "", nil, err
}
varlibPath = filepath.Join(tmpdir, "/var/lib/kubelet")
varlibPath := filepath.Join(tmpdir, "/var/lib/kubelet")
if err := os.MkdirAll(varlibPath, 0755); err != nil {
return nil, "", "", err
return nil, "", "", nil, err
}
return NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, nil
}
func TestNsenterExistsFile(t *testing.T) {
var isRoot bool
usr, err := user.Current()
if err == nil {
isRoot = usr.Username == "root"
} else {
switch err.(type) {
case user.UnknownUserIdError:
// Root should be always known, this is some random UID
isRoot = false
default:
t.Fatal(err)
}
}
tests := []struct {
name string
prepare func(base, rootfs string) (string, error)
expectedOutput bool
expectError bool
}{
{
name: "simple existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/file
path := filepath.Join(base, "file")
if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil {
return "", err
}
// In kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, path, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: true,
},
{
name: "simple non-existing file",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "file")
return path, nil
},
expectedOutput: false,
},
{
name: "simple non-accessible file",
prepare: func(base, rootfs string) (string, error) {
// On the host:
// create /base/dir/file, then make the dir inaccessible
dir := filepath.Join(base, "dir")
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
}
path := filepath.Join(dir, "file")
if err := ioutil.WriteFile(path, []byte{}, 0); err != nil {
return "", err
}
if err := os.Chmod(dir, 0644); err != nil {
return "", err
}
// In kubelet: do the same with /rootfs/base/dir/file
rootfsPath, err := writeRootfsFile(rootfs, path, 0777)
if err != nil {
return "", err
}
rootfsDir := filepath.Dir(rootfsPath)
if err := os.Chmod(rootfsDir, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: isRoot, // ExistsPath success when running as root
expectError: !isRoot, // ExistsPath must fail when running as not-root
},
{
name: "relative symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink("file", path); err != nil {
return "", err
}
// In kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: true,
},
{
name: "absolute symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> /base/file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink(file, path); err != nil {
return "", err
}
// In kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: true,
},
{
name: "relative symlink to non-existing file",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "link")
if err := os.Symlink("file", path); err != nil {
return "", err
}
return path, nil
},
expectedOutput: false,
},
{
name: "absolute symlink to non-existing file",
prepare: func(base, rootfs string) (string, error) {
file := filepath.Join(base, "file")
path := filepath.Join(base, "link")
if err := os.Symlink(file, path); err != nil {
return "", err
}
return path, nil
},
expectedOutput: false,
},
{
name: "symlink loop",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "link")
if err := os.Symlink(path, path); err != nil {
return "", err
}
return path, nil
},
expectedOutput: false,
// TODO: realpath -m is not able to detect symlink loop. Should we care?
expectError: false,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-exists-file")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
path, err := test.prepare(testBase, rootfs)
if err != nil {
t.Error(err)
continue
}
out, err := mounter.ExistsPath(path)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if err == nil && test.expectError {
t.Errorf("Test %q: expected error, got none", test.name)
}
if out != test.expectedOutput {
t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedOutput, out)
}
}
}
func TestNsenterGetMode(t *testing.T) {
tests := []struct {
name string
prepare func(base, rootfs string) (string, error)
expectedMode os.FileMode
expectError bool
}{
{
name: "simple file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/file
path := filepath.Join(base, "file")
if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil {
return "", err
}
// Prepare a different file as /rootfs/base/file (="the host
// visible from container") to check that NsEnterMounter calls
// stat on this file and not on /base/file.
// Visible from kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, path, 0777); err != nil {
return "", err
}
return path, nil
},
expectedMode: 0777,
},
{
name: "non-existing file",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "file")
return path, nil
},
expectedMode: 0,
expectError: true,
},
{
name: "absolute symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> /base/file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0644); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink(file, path); err != nil {
return "", err
}
// Visible from kubelet:
// /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0747); err != nil {
return "", err
}
return path, nil
},
expectedMode: 0747,
},
{
name: "relative symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0741); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink("file", path); err != nil {
return "", err
}
// Visible from kubelet:
// /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0647); err != nil {
return "", err
}
return path, nil
},
expectedMode: 0647,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
path, err := test.prepare(testBase, rootfs)
if err != nil {
t.Error(err)
continue
}
mode, err := mounter.GetMode(path)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if err == nil && test.expectError {
t.Errorf("Test %q: expected error, got none", test.name)
}
if mode != test.expectedMode {
t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedMode, mode)
}
}
}
func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) {
fullPath := filepath.Join(rootfs, path)
dir := filepath.Dir(fullPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
}
if err := ioutil.WriteFile(fullPath, []byte{}, mode); err != nil {
return "", err
}
// Use chmod, io.WriteFile is affected by umask
if err := os.Chmod(fullPath, mode); err != nil {
return "", err
}
return fullPath, nil
return mount.NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, ne, nil
}
func TestNsenterSafeMakeDir(t *testing.T) {
@ -659,18 +278,21 @@ func TestNsenterSafeMakeDir(t *testing.T) {
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
tmpdir, err := ioutil.TempDir("", "nsenter-get-safedir-")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t)
mounter, rootfs, varlib, ne, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
fsp := NewNSEnter(mounter, ne, varlib)
// Prepare base directory for the test
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
@ -700,7 +322,7 @@ func TestNsenterSafeMakeDir(t *testing.T) {
testBase = varlib
}
err = mounter.SafeMakeDir(test.subdir, testBase, 0755)
err = fsp.SafeMakeDir(test.subdir, testBase, 0755)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}

127
pkg/volume/util/subpath/subpath_unsupported.go
View File

@ -16,124 +16,39 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
package subpath
import (
"errors"
"os"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/utils/nsenter"
)
type Mounter struct {
mounterPath string
type subpath struct{}
var errUnsupported = errors.New("util/subpath on this platform is not supported")
// New returns a subpath.Interface for the current system.
func New(mount.Interface) Interface {
return &subpath{}
}
var unsupportedErr = errors.New("util/mount on this platform is not supported")
// New returns a mount.Interface for the current system.
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
func New(mounterPath string) Interface {
return &Mounter{
mounterPath: mounterPath,
}
// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
// OS choices. however, NSEnter is only valid on Linux
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return nil
}
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
return unsupportedErr
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, errUnsupported
}
func (mounter *Mounter) Unmount(target string) error {
return unsupportedErr
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return errUnsupported
}
func (mounter *Mounter) List() ([]MountPoint, error) {
return []MountPoint{}, unsupportedErr
}
func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
return (mp.Path == dir)
}
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return isNotMountPoint(mounter, dir)
}
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
return true, unsupportedErr
}
func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
return "", unsupportedErr
}
func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) {
return "", unsupportedErr
}
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
return false, unsupportedErr
}
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
return true, unsupportedErr
}
func (mounter *Mounter) MakeRShared(path string) error {
return unsupportedErr
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
return mounter.Interface.Mount(source, target, fstype, options)
}
func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
return true, unsupportedErr
}
func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
return FileType("fake"), unsupportedErr
}
func (mounter *Mounter) MakeDir(pathname string) error {
return unsupportedErr
}
func (mounter *Mounter) MakeFile(pathname string) error {
return unsupportedErr
}
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
return true, errors.New("not implemented")
}
func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return "", unsupportedErr
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, unsupportedErr
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return unsupportedErr
}
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return unsupportedErr
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}
func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
return -1, errors.New("not implemented")
}
func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented")
}
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return errUnsupported
}

401
pkg/volume/util/subpath/subpath_windows.go
View File

@ -16,269 +16,33 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
package subpath
import (
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"strconv"
"strings"
"syscall"
"k8s.io/klog"
utilpath "k8s.io/utils/path"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/utils/nsenter"
)
// Mounter provides the default implementation of mount.Interface
// for the windows platform. This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct {
mounterPath string
type subpath struct{}
// New returns a subpath.Interface for the current system
func New(mount.Interface) Interface {
return &subpath{}
}
// New returns a mount.Interface for the current system.
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
func New(mounterPath string) Interface {
return &Mounter{
mounterPath: mounterPath,
}
}
// Mount : mounts source to target with given options.
// currently only supports cifs(smb), bind mount(for disk)
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
target = normalizeWindowsPath(target)
if source == "tmpfs" {
klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, options)
return os.MkdirAll(target, 0755)
}
parentDir := filepath.Dir(target)
if err := os.MkdirAll(parentDir, 0755); err != nil {
return err
}
klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
options, source, target, fstype)
bindSource := source
// tell it's going to mount azure disk or azure file according to options
if bind, _, _ := isBind(options); bind {
// mount azure disk
bindSource = normalizeWindowsPath(source)
} else {
if len(options) < 2 {
klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
options, len(options), source, target)
// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
// OS choices. however, NSEnter is only valid on Linux
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return nil
}
// currently only cifs mount is supported
if strings.ToLower(fstype) != "cifs" {
return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options)
}
if output, err := newSMBMapping(options[0], options[1], source); err != nil {
if isSMBMappingExist(source) {
klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source)
if output, err := removeSMBMapping(source); err != nil {
return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output)
}
if output, err := newSMBMapping(options[0], options[1], source); err != nil {
return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output)
}
} else {
return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output)
}
}
}
if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil {
klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output))
return err
}
return nil
}
// do the SMB mount with username, password, remotepath
// return (output, error)
func newSMBMapping(username, password, remotepath string) (string, error) {
if username == "" || password == "" || remotepath == "" {
return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, password, remotepath)
}
// use PowerShell Environment Variables to store user input string to prevent command line injection
// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`
cmd := exec.Command("powershell", "/c", cmdLine)
cmd.Env = append(os.Environ(),
fmt.Sprintf("smbuser=%s", username),
fmt.Sprintf("smbpassword=%s", password),
fmt.Sprintf("smbremotepath=%s", remotepath))
output, err := cmd.CombinedOutput()
return string(output), err
}
// check whether remotepath is already mounted
func isSMBMappingExist(remotepath string) bool {
cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`)
cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
_, err := cmd.CombinedOutput()
return err == nil
}
// remove SMB mapping
func removeSMBMapping(remotepath string) (string, error) {
cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`)
cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
output, err := cmd.CombinedOutput()
return string(output), err
}
// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
klog.V(4).Infof("azureMount: Unmount target (%q)", target)
target = normalizeWindowsPath(target)
if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil {
klog.Errorf("rmdir failed: %v, output: %q", err, string(output))
return err
}
return nil
}
// List returns a list of all mounted filesystems. todo
func (mounter *Mounter) List() ([]MountPoint, error) {
return []MountPoint{}, nil
}
// IsMountPointMatch determines if the mountpoint matches the dir
func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
return mp.Path == dir
}
// IsNotMountPoint determines if a directory is a mountpoint.
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return isNotMountPoint(mounter, dir)
}
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
stat, err := os.Lstat(file)
if err != nil {
return true, err
}
// If current file is a symlink, then it is a mountpoint.
if stat.Mode()&os.ModeSymlink != 0 {
target, err := os.Readlink(file)
if err != nil {
return true, fmt.Errorf("readlink error: %v", err)
}
exists, err := mounter.ExistsPath(target)
if err != nil {
return true, err
}
return !exists, nil
}
return true, nil
}
// GetDeviceNameFromMount given a mnt point, find the device
func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
return getDeviceNameFromMount(mounter, mountPath, pluginDir)
}
// getDeviceNameFromMount find the device(drive) name in which
// the mount path reference should match the given plugin directory. In case no mount path reference
// matches, returns the volume name taken from its given mountPath
func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) {
refs, err := mounter.GetMountRefs(mountPath)
if err != nil {
klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
return "", err
}
if len(refs) == 0 {
return "", fmt.Errorf("directory %s is not mounted", mountPath)
}
basemountPath := normalizeWindowsPath(path.Join(pluginDir, MountsInGlobalPDPath))
for _, ref := range refs {
if strings.Contains(ref, basemountPath) {
volumeID, err := filepath.Rel(normalizeWindowsPath(basemountPath), ref)
if err != nil {
klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
return "", err
}
return volumeID, nil
}
}
return path.Base(mountPath), nil
}
// DeviceOpened determines if the device is in use elsewhere
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
return false, nil
}
// PathIsDevice determines if a path is a device.
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
return false, nil
}
// MakeRShared checks that given path is on a mount with 'rshared' mount
// propagation. Empty implementation here.
func (mounter *Mounter) MakeRShared(path string) error {
return nil
}
// GetFileType checks for sockets/block/character devices
func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
return getFileType(pathname)
}
// MakeFile creates a new directory
func (mounter *Mounter) MakeDir(pathname string) error {
err := os.MkdirAll(pathname, os.FileMode(0755))
if err != nil {
if !os.IsExist(err) {
return err
}
}
return nil
}
// MakeFile creates an empty file
func (mounter *Mounter) MakeFile(pathname string) error {
f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644))
defer f.Close()
if err != nil {
if !os.IsExist(err) {
return err
}
}
return nil
}
// ExistsPath checks whether the path exists
func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}
// EvalHostSymlinks returns the path name after evaluating symlinks
func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return filepath.EvalSymlinks(pathname)
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
@ -310,7 +74,7 @@ func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, e
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if startsWithBackstep(relSubPath) {
if mount.StartsWithBackstep(relSubPath) {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
@ -345,7 +109,7 @@ func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, e
break
}
if !PathWithinBase(currentFullPath, volumePath) {
if !mount.PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
@ -381,7 +145,7 @@ func lockPath(path string) (uintptr, error) {
}
// Lock all directories in subPath and check they're not symlinks.
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
@ -392,139 +156,12 @@ func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string,
}
// No bind-mounts for subpaths are necessary on Windows
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
// Try to mount the disk
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
if err := ValidateDiskNumber(source); err != nil {
klog.Errorf("diskMount: formatAndMount failed, err: %v", err)
return err
}
if len(fstype) == 0 {
// Use 'NTFS' as the default
fstype = "NTFS"
}
// format disk if it is unformatted(raw)
cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+
" | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype)
if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil {
return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output))
}
klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype)
driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec)
if err != nil {
return err
}
driverPath := driveLetter + ":"
target = normalizeWindowsPath(target)
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target)
if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil {
klog.Errorf("mklink failed: %v, output: %q", err, string(output))
return err
}
return nil
}
func normalizeWindowsPath(path string) string {
normalizedPath := strings.Replace(path, "/", "\\", -1)
if strings.HasPrefix(normalizedPath, "\\") {
normalizedPath = "c:" + normalizedPath
}
return normalizedPath
}
// ValidateDiskNumber : disk number should be a number in [0, 99]
func ValidateDiskNumber(disk string) error {
diskNum, err := strconv.Atoi(disk)
if err != nil {
return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err)
}
if diskNum < 0 || diskNum > 99 {
return fmt.Errorf("disk number out of range: %q", disk)
}
return nil
}
// Get drive letter according to windows disk number
func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) {
cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum)
output, err := exec.Run("powershell", "/c", cmd)
if err != nil {
return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output))
}
if len(string(output)) < 1 {
return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty")
}
return string(output)[:1], nil
}
// getAllParentLinks walks all symbolic links and return all the parent targets recursively
func getAllParentLinks(path string) ([]string, error) {
const maxIter = 255
links := []string{}
for {
links = append(links, path)
if len(links) > maxIter {
return links, fmt.Errorf("unexpected length of parent links: %v", links)
}
fi, err := os.Lstat(path)
if err != nil {
return links, fmt.Errorf("Lstat: %v", err)
}
if fi.Mode()&os.ModeSymlink == 0 {
break
}
path, err = os.Readlink(path)
if err != nil {
return links, fmt.Errorf("Readlink error: %v", err)
}
}
return links, nil
}
// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) {
return []string{}, nil
} else if err != nil {
return nil, err
}
return []string{pathname}, nil
}
// Note that on windows, it always returns 0. We actually don't set FSGroup on
// windows platform, see SetVolumeOwnership implementation.
func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
return 0, nil
}
func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
// Windows does not support SELinux.
return false, nil
}
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
info, err := os.Stat(pathname)
if err != nil {
return 0, err
}
return info.Mode(), nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
@ -537,7 +174,7 @@ func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
if !mount.PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
@ -572,7 +209,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !PathWithinBase(fullExistingPath, fullBasePath) {
if !mount.PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
@ -619,7 +256,7 @@ func findExistingPrefix(base, pathname string) (string, []string, error) {
return base, nil, err
}
if startsWithBackstep(rel) {
if mount.StartsWithBackstep(rel) {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}

389
pkg/volume/util/subpath/subpath_windows_test.go
View File

@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
package subpath
import (
"fmt"
@ -24,54 +24,11 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestNormalizeWindowsPath(t *testing.T) {
path := `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4/volumes/kubernetes.io~azure-disk`
normalizedPath := normalizeWindowsPath(path)
if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` {
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
}
path = `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk`
normalizedPath = normalizeWindowsPath(path)
if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` {
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
}
path = `/`
normalizedPath = normalizeWindowsPath(path)
if normalizedPath != `c:\` {
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
}
}
func TestValidateDiskNumber(t *testing.T) {
diskNum := "0"
if err := ValidateDiskNumber(diskNum); err != nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
diskNum = "99"
if err := ValidateDiskNumber(diskNum); err != nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
diskNum = "ab"
if err := ValidateDiskNumber(diskNum); err == nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
diskNum = "100"
if err := ValidateDiskNumber(diskNum); err == nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
}
func makeLink(link, target string) error {
if output, err := exec.Command("cmd", "/c", "mklink", "/D", link, target).CombinedOutput(); err != nil {
return fmt.Errorf("mklink failed: %v, link(%q) target(%q) output: %q", err, link, target, string(output))
@ -79,61 +36,6 @@ func makeLink(link, target string) error {
return nil
}
func removeLink(link string) error {
if output, err := exec.Command("cmd", "/c", "rmdir", link).CombinedOutput(); err != nil {
return fmt.Errorf("rmdir failed: %v, output: %q", err, string(output))
}
return nil
}
func setEquivalent(set1, set2 []string) bool {
map1 := make(map[string]bool)
map2 := make(map[string]bool)
for _, s := range set1 {
map1[s] = true
}
for _, s := range set2 {
map2[s] = true
}
for s := range map1 {
if !map2[s] {
return false
}
}
for s := range map2 {
if !map1[s] {
return false
}
}
return true
}
// this func must run in admin mode, otherwise it will fail
func TestGetMountRefs(t *testing.T) {
tests := []struct {
mountPath string
expectedRefs []string
}{
{
mountPath: `c:\windows`,
expectedRefs: []string{`c:\windows`},
},
{
mountPath: `c:\doesnotexist`,
expectedRefs: []string{},
},
}
mounter := Mounter{"fake/path"}
for _, test := range tests {
if refs, err := mounter.GetMountRefs(test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) {
t.Errorf("getMountRefs(%q) = %v, error: %v; expected %v", test.mountPath, refs, err, test.expectedRefs)
}
}
}
func TestDoSafeMakeDir(t *testing.T) {
base, err := ioutil.TempDir("", "TestDoSafeMakeDir")
if err != nil {
@ -536,292 +438,3 @@ func TestFindExistingPrefix(t *testing.T) {
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestPathWithinBase(t *testing.T) {
tests := []struct {
fullPath string
basePath string
expectedResult bool
}{
{
fullPath: `c:\tmp\a\b\c`,
basePath: `c:\tmp`,
expectedResult: true,
},
{
fullPath: `c:\tmp1`,
basePath: `c:\tmp2`,
expectedResult: false,
},
{
fullPath: `c:\tmp`,
basePath: `c:\tmp`,
expectedResult: true,
},
{
fullPath: `c:\tmp`,
basePath: `c:\tmp\a\b\c`,
expectedResult: false,
},
{
fullPath: `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config\..timestamp\file.txt`,
basePath: `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config`,
expectedResult: true,
},
}
for _, test := range tests {
result := PathWithinBase(test.fullPath, test.basePath)
assert.Equal(t, result, test.expectedResult, "Expect result not equal with PathWithinBase(%s, %s) return: %q, expected: %q",
test.fullPath, test.basePath, result, test.expectedResult)
}
}
func TestGetFileType(t *testing.T) {
mounter := New("fake/path")
testCase := []struct {
name string
expectedType FileType
setUp func() (string, string, error)
}{
{
"Directory Test",
FileTypeDirectory,
func() (string, string, error) {
tempDir, err := ioutil.TempDir("", "test-get-filetype-")
return tempDir, tempDir, err
},
},
{
"File Test",
FileTypeFile,
func() (string, string, error) {
tempFile, err := ioutil.TempFile("", "test-get-filetype")
if err != nil {
return "", "", err
}
tempFile.Close()
return tempFile.Name(), tempFile.Name(), nil
},
},
}
for idx, tc := range testCase {
path, cleanUpPath, err := tc.setUp()
if err != nil {
t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err)
}
if len(cleanUpPath) > 0 {
defer os.RemoveAll(cleanUpPath)
}
fileType, err := mounter.GetFileType(path)
if err != nil {
t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err)
}
if fileType != tc.expectedType {
t.Fatalf("[%d-%s] expected %s, but got %s", idx, tc.name, tc.expectedType, fileType)
}
}
}
func TestIsLikelyNotMountPoint(t *testing.T) {
mounter := Mounter{"fake/path"}
tests := []struct {
fileName string
targetLinkName string
setUp func(base, fileName, targetLinkName string) error
expectedResult bool
expectError bool
}{
{
"Dir",
"",
func(base, fileName, targetLinkName string) error {
return os.Mkdir(filepath.Join(base, fileName), 0750)
},
true,
false,
},
{
"InvalidDir",
"",
func(base, fileName, targetLinkName string) error {
return nil
},
true,
true,
},
{
"ValidSymLink",
"targetSymLink",
func(base, fileName, targetLinkName string) error {
targeLinkPath := filepath.Join(base, targetLinkName)
if err := os.Mkdir(targeLinkPath, 0750); err != nil {
return err
}
filePath := filepath.Join(base, fileName)
if err := makeLink(filePath, targeLinkPath); err != nil {
return err
}
return nil
},
false,
false,
},
{
"InvalidSymLink",
"targetSymLink2",
func(base, fileName, targetLinkName string) error {
targeLinkPath := filepath.Join(base, targetLinkName)
if err := os.Mkdir(targeLinkPath, 0750); err != nil {
return err
}
filePath := filepath.Join(base, fileName)
if err := makeLink(filePath, targeLinkPath); err != nil {
return err
}
return removeLink(targeLinkPath)
},
true,
false,
},
}
for _, test := range tests {
base, err := ioutil.TempDir("", test.fileName)
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
if err := test.setUp(base, test.fileName, test.targetLinkName); err != nil {
t.Fatalf("unexpected error in setUp(%s, %s): %v", test.fileName, test.targetLinkName, err)
}
filePath := filepath.Join(base, test.fileName)
result, err := mounter.IsLikelyNotMountPoint(filePath)
assert.Equal(t, result, test.expectedResult, "Expect result not equal with IsLikelyNotMountPoint(%s) return: %q, expected: %q",
filePath, result, test.expectedResult)
if test.expectError {
assert.NotNil(t, err, "Expect error during IsLikelyNotMountPoint(%s)", filePath)
} else {
assert.Nil(t, err, "Expect error is nil during IsLikelyNotMountPoint(%s)", filePath)
}
}
}
func TestFormatAndMount(t *testing.T) {
fakeMounter := ErrorMounter{&FakeMounter{}, 0, nil}
execCallback := func(cmd string, args ...string) ([]byte, error) {
for j := range args {
if strings.Contains(args[j], "Get-Disk -Number") {
return []byte("0"), nil
}
if strings.Contains(args[j], "Get-Partition -DiskNumber") {
return []byte("0"), nil
}
if strings.Contains(args[j], "mklink") {
return nil, nil
}
}
return nil, fmt.Errorf("Unexpected cmd %s, args %v", cmd, args)
}
fakeExec := NewFakeExec(execCallback)
mounter := SafeFormatAndMount{
Interface: &fakeMounter,
Exec: fakeExec,
}
tests := []struct {
device string
target string
fstype string
mountOptions []string
expectError bool
}{
{
"0",
"disk",
"NTFS",
[]string{},
false,
},
{
"0",
"disk",
"",
[]string{},
false,
},
{
"invalidDevice",
"disk",
"NTFS",
[]string{},
true,
},
}
for _, test := range tests {
base, err := ioutil.TempDir("", test.device)
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
target := filepath.Join(base, test.target)
err = mounter.FormatAndMount(test.device, target, test.fstype, test.mountOptions)
if test.expectError {
assert.NotNil(t, err, "Expect error during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions)
} else {
assert.Nil(t, err, "Expect error is nil during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions)
}
}
}
func TestNewSMBMapping(t *testing.T) {
tests := []struct {
username string
password string
remotepath string
expectError bool
}{
{
"",
"password",
`\\remotepath`,
true,
},
{
"username",
"",
`\\remotepath`,
true,
},
{
"username",
"password",
"",
true,
},
}
for _, test := range tests {
_, err := newSMBMapping(test.username, test.password, test.remotepath)
if test.expectError {
assert.NotNil(t, err, "Expect error during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath)
} else {
assert.Nil(t, err, "Expect error is nil during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath)
}
}
}