diff --git a/pkg/cloudprovider/providers/BUILD b/pkg/cloudprovider/providers/BUILD index ac4cc7fac12..6be8a179d17 100644 --- a/pkg/cloudprovider/providers/BUILD +++ b/pkg/cloudprovider/providers/BUILD @@ -16,10 +16,10 @@ go_library( "//cmd/kubelet/app:__pkg__", ], deps = [ - "//pkg/cloudprovider/providers/openstack:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/aws:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/gce:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/vsphere:go_default_library", ], ) @@ -33,10 +33,7 @@ filegroup( filegroup( name = "all-srcs", - srcs = [ - ":package-srcs", - "//pkg/cloudprovider/providers/openstack:all-srcs", - ], + srcs = [":package-srcs"], tags = ["automanaged"], ) diff --git a/pkg/cloudprovider/providers/providers.go b/pkg/cloudprovider/providers/providers.go index f258be3134f..8213c4802ca 100644 --- a/pkg/cloudprovider/providers/providers.go +++ b/pkg/cloudprovider/providers/providers.go @@ -20,9 +20,9 @@ package cloudprovider import ( // Cloud providers - _ "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" _ "k8s.io/legacy-cloud-providers/aws" _ "k8s.io/legacy-cloud-providers/azure" _ "k8s.io/legacy-cloud-providers/gce" + _ "k8s.io/legacy-cloud-providers/openstack" _ "k8s.io/legacy-cloud-providers/vsphere" ) diff --git a/pkg/volume/cinder/BUILD b/pkg/volume/cinder/BUILD index e4c740756fb..a51e18c3d9f 100644 --- a/pkg/volume/cinder/BUILD +++ b/pkg/volume/cinder/BUILD @@ -17,7 +17,6 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/volume/cinder", deps = [ - "//pkg/cloudprovider/providers/openstack:go_default_library", "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", @@ -33,6 +32,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/keymutex:go_default_library", @@ -49,7 +49,6 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/cloudprovider/providers/openstack:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", @@ -60,6 +59,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index 3d37dd86a09..040ea619789 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -25,18 +25,18 @@ import ( "path" "path/filepath" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" - "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/legacy-cloud-providers/openstack" "k8s.io/utils/keymutex" utilstrings "k8s.io/utils/strings" ) diff --git a/pkg/volume/cinder/cinder_test.go b/pkg/volume/cinder/cinder_test.go index ec1dbee7118..244a2cf8153 100644 --- a/pkg/volume/cinder/cinder_test.go +++ b/pkg/volume/cinder/cinder_test.go @@ -28,11 +28,11 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utiltesting "k8s.io/client-go/util/testing" - "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/legacy-cloud-providers/openstack" ) func TestCanSupport(t *testing.T) { diff --git a/staging/src/k8s.io/legacy-cloud-providers/BUILD b/staging/src/k8s.io/legacy-cloud-providers/BUILD index be967137ea3..65be89d13be 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/BUILD @@ -12,6 +12,7 @@ filegroup( "//staging/src/k8s.io/legacy-cloud-providers/aws:all-srcs", "//staging/src/k8s.io/legacy-cloud-providers/azure:all-srcs", "//staging/src/k8s.io/legacy-cloud-providers/gce:all-srcs", + "//staging/src/k8s.io/legacy-cloud-providers/openstack:all-srcs", "//staging/src/k8s.io/legacy-cloud-providers/vsphere:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/cloudprovider/providers/openstack/BUILD b/staging/src/k8s.io/legacy-cloud-providers/openstack/BUILD similarity index 91% rename from pkg/cloudprovider/providers/openstack/BUILD rename to staging/src/k8s.io/legacy-cloud-providers/openstack/BUILD index 6ce8090d2d3..925fcacbec2 100644 --- a/pkg/cloudprovider/providers/openstack/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/BUILD @@ -19,9 +19,9 @@ go_library( "openstack_routes.go", "openstack_volumes.go", ], - importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack", + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack", + importpath = "k8s.io/legacy-cloud-providers/openstack", deps = [ - "//pkg/util/mount:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -40,6 +40,7 @@ go_library( "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount:go_default_library", "//vendor/github.com/gophercloud/gophercloud:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library", @@ -103,6 +104,11 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/file:all-srcs", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/io:all-srcs", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount:all-srcs", + ], tags = ["automanaged"], ) diff --git a/pkg/cloudprovider/providers/openstack/MAINTAINERS.md b/staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md similarity index 59% rename from pkg/cloudprovider/providers/openstack/MAINTAINERS.md rename to staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md index f71afec99fb..072e071c6b9 100644 --- a/pkg/cloudprovider/providers/openstack/MAINTAINERS.md +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md @@ -3,4 +3,4 @@ * [Angus Lees](https://github.com/anguslees) -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/pkg/cloudprovider/providers/openstack/MAINTAINERS.md?pixel)]() +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/staging/src/k8s.io/legacy-cloud-providers/openstack/MAINTAINERS.md?pixel)]() diff --git a/pkg/cloudprovider/providers/openstack/OWNERS b/staging/src/k8s.io/legacy-cloud-providers/openstack/OWNERS similarity index 100% rename from pkg/cloudprovider/providers/openstack/OWNERS rename to staging/src/k8s.io/legacy-cloud-providers/openstack/OWNERS diff --git a/pkg/cloudprovider/providers/openstack/metadata.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/metadata.go similarity index 98% rename from pkg/cloudprovider/providers/openstack/metadata.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/metadata.go index f00e0e1c898..7562a684562 100644 --- a/pkg/cloudprovider/providers/openstack/metadata.go +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/metadata.go @@ -30,7 +30,7 @@ import ( "strings" "k8s.io/klog" - "k8s.io/kubernetes/pkg/util/mount" + osmount "k8s.io/legacy-cloud-providers/openstack/util/mount" "k8s.io/utils/exec" ) @@ -125,7 +125,7 @@ func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) { klog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir) - mounter := mount.New("" /* default mount path */) + mounter := osmount.New("" /* default mount path */) err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"}) if err != nil { err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"}) diff --git a/pkg/cloudprovider/providers/openstack/metadata_test.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/metadata_test.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/metadata_test.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/metadata_test.go diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_client.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_client.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_client.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_client.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_instances.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_instances.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_instances.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_instances.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_loadbalancer.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_loadbalancer.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_metrics.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_metrics.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_metrics.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_metrics.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_routes.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes_test.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes_test.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_routes_test.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_routes_test.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_test.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_test.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_test.go diff --git a/pkg/cloudprovider/providers/openstack/openstack_volumes.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_volumes.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_volumes.go rename to staging/src/k8s.io/legacy-cloud-providers/openstack/openstack_volumes.go diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/file/BUILD b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/file/BUILD new file mode 100644 index 00000000000..e464bc43fdc --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/file/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["file.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack/util/file", + importpath = "k8s.io/legacy-cloud-providers/openstack/util/file", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/file/file.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/file/file.go new file mode 100644 index 00000000000..70d26c4ef74 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/file/file.go @@ -0,0 +1,57 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package file + +import ( + "os" +) + +// FileExists checks if specified file exists. +func FileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + +// FileOrSymlinkExists checks if specified file or symlink exists. +func FileOrSymlinkExists(filename string) (bool, error) { + if _, err := os.Lstat(filename); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + +// ReadDirNoStat returns a string of files/directories contained +// in dirname without calling lstat on them. +func ReadDirNoStat(dirname string) ([]string, error) { + if dirname == "" { + dirname = "." + } + + f, err := os.Open(dirname) + if err != nil { + return nil, err + } + defer f.Close() + + return f.Readdirnames(-1) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/io/BUILD b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/io/BUILD new file mode 100644 index 00000000000..c80afc4d210 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/io/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["consistentread.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack/util/io", + importpath = "k8s.io/legacy-cloud-providers/openstack/util/io", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/io/consistentread.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/io/consistentread.go new file mode 100644 index 00000000000..6e1f17b0985 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/io/consistentread.go @@ -0,0 +1,45 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package io + +import ( + "bytes" + "fmt" + "io/ioutil" +) + +// ConsistentRead repeatedly reads a file until it gets the same content twice. +// This is useful when reading files in /proc that are larger than page size +// and kernel may modify them between individual read() syscalls. +func ConsistentRead(filename string, attempts int) ([]byte, error) { + oldContent, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + for i := 0; i < attempts; i++ { + newContent, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + if bytes.Compare(oldContent, newContent) == 0 { + return newContent, nil + } + // Files are different, continue reading + oldContent = newContent + } + return nil, fmt.Errorf("could not get consistent content of %s after %d attempts", filename, attempts) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/BUILD b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/BUILD new file mode 100644 index 00000000000..2fc723ebedb --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/BUILD @@ -0,0 +1,43 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "mount.go", + "mount_linux.go", + "mount_unsupported.go", + "mount_windows.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/openstack/util/mount", + importpath = "k8s.io/legacy-cloud-providers/openstack/util/mount", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/file:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/io:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//staging/src/k8s.io/legacy-cloud-providers/openstack/util/file:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount.go new file mode 100644 index 00000000000..a1a874bdce0 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount.go @@ -0,0 +1,406 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// TODO(thockin): This whole pkg is pretty linux-centric. As soon as we have +// an alternate platform, we will need to abstract further. +package mount + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type FileType string + +const ( + // Default mount command if mounter path is not specified + defaultMountCommand = "mount" + MountsInGlobalPDPath = "mounts" + FileTypeDirectory FileType = "Directory" + FileTypeFile FileType = "File" + FileTypeSocket FileType = "Socket" + FileTypeCharDev FileType = "CharDevice" + FileTypeBlockDev FileType = "BlockDevice" +) + +type Interface interface { + // Mount mounts source to target as fstype with given options. + Mount(source string, target string, fstype string, options []string) error + // Unmount unmounts given target. + Unmount(target string) error + // List returns a list of all mounted filesystems. This can be large. + // On some platforms, reading mounts is not guaranteed consistent (i.e. + // it could change between chunked reads). This is guaranteed to be + // consistent. + List() ([]MountPoint, error) + // IsMountPointMatch determines if the mountpoint matches the dir + IsMountPointMatch(mp MountPoint, dir string) bool + // IsNotMountPoint determines if a directory is a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsNotMountPoint is more expensive than IsLikelyNotMountPoint. + // IsNotMountPoint detects bind mounts in linux. + // IsNotMountPoint enumerates all the mountpoints using List() and + // the list of mountpoints may be large, then it uses + // IsMountPointMatch to evaluate whether the directory is a mountpoint + IsNotMountPoint(file string) (bool, error) + // IsLikelyNotMountPoint uses heuristics to determine if a directory + // is a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsLikelyNotMountPoint does NOT properly detect all mountpoint types + // most notably linux bind mounts. + IsLikelyNotMountPoint(file string) (bool, error) + // DeviceOpened determines if the device is in use elsewhere + // on the system, i.e. still mounted. + DeviceOpened(pathname string) (bool, error) + // PathIsDevice determines if a path is a device. + PathIsDevice(pathname string) (bool, error) + // GetDeviceNameFromMount finds the device name by checking the mount path + // to get the global mount path which matches its plugin directory + GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) + // MakeRShared checks that given path is on a mount with 'rshared' mount + // propagation. If not, it bind-mounts the path as rshared. + MakeRShared(path string) error + // GetFileType checks for file/directory/socket/block/character devices. + // Will operate in the host mount namespace if kubelet is running in a container + GetFileType(pathname string) (FileType, error) + // MakeFile creates an empty file. + // Will operate in the host mount namespace if kubelet is running in a container + MakeFile(pathname string) error + // MakeDir creates a new directory. + // Will operate in the host mount namespace if kubelet is running in a container + MakeDir(pathname string) error + // SafeMakeDir creates subdir within given base. It makes sure that the + // created directory does not escape given base directory mis-using + // symlinks. Note that the function makes sure that it creates the directory + // somewhere under the base, nothing else. E.g. if the directory already + // exists, it may exist outside of the base due to symlinks. + // This method should be used if the directory to create is inside volume + // that's under user control. User must not be able to use symlinks to + // escape the volume to create directories somewhere else. + SafeMakeDir(subdir string, base string, perm os.FileMode) error + // Will operate in the host mount namespace if kubelet is running in a container. + // Error is returned on any other error than "file not found". + ExistsPath(pathname string) (bool, error) + // EvalHostSymlinks returns the path name after evaluating symlinks. + // Will operate in the host mount namespace if kubelet is running in a container. + EvalHostSymlinks(pathname string) (string, error) + // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given + // pod volume directory. + CleanSubPaths(podDir string, volumeName string) error + // PrepareSafeSubpath does everything that's necessary to prepare a subPath + // that's 1) inside given volumePath and 2) immutable after this call. + // + // newHostPath - location of prepared subPath. It should be used instead of + // hostName when running the container. + // cleanupAction - action to run when the container is running or it failed to start. + // + // CleanupAction must be called immediately after the container with given + // subpath starts. On the other hand, Interface.CleanSubPaths must be called + // when the pod finishes. + PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) + // GetMountRefs finds all mount references to the path, returns a + // list of paths. Path could be a mountpoint path, device or a normal + // directory (for bind mount). + GetMountRefs(pathname string) ([]string, error) + // GetFSGroup returns FSGroup of the path. + GetFSGroup(pathname string) (int64, error) + // GetSELinuxSupport returns true if given path is on a mount that supports + // SELinux. + GetSELinuxSupport(pathname string) (bool, error) + // GetMode returns permissions of the path. + GetMode(pathname string) (os.FileMode, error) +} + +type Subpath struct { + // index of the VolumeMount for this container + VolumeMountIndex int + // Full path to the subpath directory on the host + Path string + // name of the volume that is a valid directory name. + VolumeName string + // Full path to the volume path + VolumePath string + // Path to the pod's directory, including pod UID + PodDir string + // Name of the container + ContainerName string +} + +// Exec executes command where mount utilities are. This can be either the host, +// container where kubelet runs or even a remote pod with mount utilities. +// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does +// not provide stdin/stdout/stderr streams. +type Exec interface { + // Run executes a command and returns its stdout + stderr combined in one + // stream. + Run(cmd string, args ...string) ([]byte, error) +} + +// Compile-time check to ensure all Mounter implementations satisfy +// the mount interface +var _ Interface = &Mounter{} + +// This represents a single line in /proc/mounts or /etc/fstab. +type MountPoint struct { + Device string + Path string + Type string + Opts []string + Freq int + Pass int +} + +// SafeFormatAndMount probes a device to see if it is formatted. +// Namely it checks to see if a file system is present. If so it +// mounts it otherwise the device is formatted first then mounted. +type SafeFormatAndMount struct { + Interface + Exec +} + +// FormatAndMount formats the given disk, if needed, and mounts it. +// That is if the disk is not formatted and it is not being mounted as +// read-only it will format it first then mount it. Otherwise, if the +// disk is already formatted or it is being mounted as read-only, it +// will be mounted without formatting. +func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error { + return mounter.formatAndMount(source, target, fstype, options) +} + +// getMountRefsByDev finds all references to the device provided +// by mountPath; returns a list of paths. +// Note that mountPath should be path after the evaluation of any symbolic links. +func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { + mps, err := mounter.List() + if err != nil { + return nil, err + } + + // Finding the device mounted to mountPath + diskDev := "" + for i := range mps { + if mountPath == mps[i].Path { + diskDev = mps[i].Device + break + } + } + + // Find all references to the device. + var refs []string + for i := range mps { + if mps[i].Device == diskDev || mps[i].Device == mountPath { + if mps[i].Path != mountPath { + refs = append(refs, mps[i].Path) + } + } + } + return refs, nil +} + +// GetDeviceNameFromMount: given a mnt point, find the device from /proc/mounts +// returns the device name, reference count, and error code +func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { + mps, err := mounter.List() + if err != nil { + return "", 0, err + } + + // Find the device name. + // FIXME if multiple devices mounted on the same mount path, only the first one is returned + device := "" + // If mountPath is symlink, need get its target path. + slTarget, err := filepath.EvalSymlinks(mountPath) + if err != nil { + slTarget = mountPath + } + for i := range mps { + if mps[i].Path == slTarget { + device = mps[i].Device + break + } + } + + // Find all references to the device. + refCount := 0 + for i := range mps { + if mps[i].Device == device { + refCount++ + } + } + return device, refCount, nil +} + +// IsNotMountPoint determines if a directory is a mountpoint. +// It should return ErrNotExist when the directory does not exist. +// This method uses the List() of all mountpoints +// It is more extensive than IsLikelyNotMountPoint +// and it detects bind mounts in linux +func IsNotMountPoint(mounter Interface, file string) (bool, error) { + // IsLikelyNotMountPoint provides a quick check + // to determine whether file IS A mountpoint + notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file) + if notMntErr != nil && os.IsPermission(notMntErr) { + // We were not allowed to do the simple stat() check, e.g. on NFS with + // root_squash. Fall back to /proc/mounts check below. + notMnt = true + notMntErr = nil + } + if notMntErr != nil { + return notMnt, notMntErr + } + // identified as mountpoint, so return this fact + if notMnt == false { + return notMnt, nil + } + + // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts + resolvedFile, err := mounter.EvalHostSymlinks(file) + if err != nil { + return true, err + } + + // check all mountpoints since IsLikelyNotMountPoint + // is not reliable for some mountpoint types + mountPoints, mountPointsErr := mounter.List() + if mountPointsErr != nil { + return notMnt, mountPointsErr + } + for _, mp := range mountPoints { + if mounter.IsMountPointMatch(mp, resolvedFile) { + notMnt = false + break + } + } + return notMnt, nil +} + +// isBind detects whether a bind mount is being requested and makes the remount options to +// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. +// The list equals: +// options - 'bind' + 'remount' (no duplicate) +func isBind(options []string) (bool, []string, []string) { + // Because we have an FD opened on the subpath bind mount, the "bind" option + // needs to be included, otherwise the mount target will error as busy if you + // remount as readonly. + // + // As a consequence, all read only bind mounts will no longer change the underlying + // volume mount to be read only. + bindRemountOpts := []string{"bind", "remount"} + bind := false + bindOpts := []string{"bind"} + + // _netdev is a userspace mount option and does not automatically get added when + // bind mount is created and hence we must carry it over. + if checkForNetDev(options) { + bindOpts = append(bindOpts, "_netdev") + } + + for _, option := range options { + switch option { + case "bind": + bind = true + break + case "remount": + break + default: + bindRemountOpts = append(bindRemountOpts, option) + } + } + + return bind, bindOpts, bindRemountOpts +} + +func checkForNetDev(options []string) bool { + for _, option := range options { + if option == "_netdev" { + return true + } + } + return false +} + +// TODO: this is a workaround for the unmount device issue caused by gci mounter. +// In GCI cluster, if gci mounter is used for mounting, the container started by mounter +// script will cause additional mounts created in the container. Since these mounts are +// irrelevant to the original mounts, they should be not considered when checking the +// mount references. Current solution is to filter out those mount paths that contain +// the string of original mount path. +// Plan to work on better approach to solve this issue. + +func HasMountRefs(mountPath string, mountRefs []string) bool { + count := 0 + for _, ref := range mountRefs { + if !strings.Contains(ref, mountPath) { + count = count + 1 + } + } + return count > 0 +} + +// PathWithinBase checks if give path is within given base directory. +func PathWithinBase(fullPath, basePath string) bool { + rel, err := filepath.Rel(basePath, fullPath) + if err != nil { + return false + } + if startsWithBackstep(rel) { + // Needed to escape the base path + return false + } + return true +} + +// startsWithBackstep checks if the given path starts with a backstep segment +func startsWithBackstep(rel string) bool { + // normalize to / and check for ../ + return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") +} + +// getFileType checks for file/directory/socket and block/character devices +func getFileType(pathname string) (FileType, error) { + var pathType FileType + info, err := os.Stat(pathname) + if os.IsNotExist(err) { + return pathType, fmt.Errorf("path %q does not exist", pathname) + } + // err in call to os.Stat + if err != nil { + return pathType, err + } + + // checks whether the mode is the target mode + isSpecificMode := func(mode, targetMode os.FileMode) bool { + return mode&targetMode == targetMode + } + + mode := info.Mode() + if mode.IsDir() { + return FileTypeDirectory, nil + } else if mode.IsRegular() { + return FileTypeFile, nil + } else if isSpecificMode(mode, os.ModeSocket) { + return FileTypeSocket, nil + } else if isSpecificMode(mode, os.ModeDevice) { + if isSpecificMode(mode, os.ModeCharDevice) { + return FileTypeCharDev, nil + } + return FileTypeBlockDev, nil + } + + return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device") +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_linux.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_linux.go new file mode 100644 index 00000000000..d83c82347e1 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_linux.go @@ -0,0 +1,1331 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/util/sets" + utilfile "k8s.io/cloud-provider-openstack/pkg/util/file" + utilio "k8s.io/cloud-provider-openstack/pkg/util/io" + "k8s.io/klog" + utilexec "k8s.io/utils/exec" +) + +const ( + // How many times to retry for a consistent read of /proc/mounts. + maxListTries = 3 + // Number of fields per line in /proc/mounts as per the fstab man page. + expectedNumFieldsPerLine = 6 + // At least number of fields per line in /proc//mountinfo. + expectedAtLeastNumFieldsPerMountInfo = 10 + // Location of the mount file to use + procMountsPath = "/proc/mounts" + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 + + // place for subpath mounts + containerSubPathDirectoryName = "volume-subpaths" + // syscall.Openat flags used to traverse directories not following symlinks + nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW + // flags for getting file descriptor without following the symlink + openFDFlags = unix.O_NOFOLLOW | unix.O_PATH +) + +// Mounter provides the default implementation of mount.Interface +// for the linux platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string + withSystemd bool +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + withSystemd: detectSystemd(), + } +} + +// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must +// be an empty string in case it's not required, e.g. for remount, or for auto filesystem +// type, where kernel handles fstype for you. The mount 'options' is a list of options, +// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is +// required, call Mount with an empty string list or nil. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts := isBind(options) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs") + if fsTypesNeedMounter.Has(fstype) { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error { + mountArgs := makeMountArgs(source, target, fstype, options) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountCmd = mounterPath + } + + if m.withSystemd { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs = addSystemdScope("systemd-run", target, mountCmd, mountArgs) + } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + args := strings.Join(mountArgs, " ") + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", + err, mountCmd, args, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Info("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) + return false + } + klog.V(2).Info("Detected OS with systemd") + return true +} + +// makeMountArgs makes the arguments to the mount(8) command. +func makeMountArgs(source, target, fstype string, options []string) []string { + // Build mount command as follows: + // mount [-t $fstype] [-o $options] [$source] $target + mountArgs := []string{} + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + } + if len(options) > 0 { + mountArgs = append(mountArgs, "-o", strings.Join(options, ",")) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + } + mountArgs = append(mountArgs, target) + + return mountArgs +} + +// addSystemdScope adds "system-run --scope" to given command line +func addSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...) +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("Unmounting %s", target) + command := exec.Command("umount", target) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("Unmount failed: %v\nUnmounting arguments: %s\nOutput: %s\n", err, target, string(output)) + } + return nil +} + +// List returns a list of all mounted filesystems. +func (*Mounter) List() ([]MountPoint, error) { + return listProcMounts(procMountsPath) +} + +func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { + deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) + return ((mp.Path == dir) || (mp.Path == deletedDir)) +} + +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return IsNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +// It is fast but not necessarily ALWAYS correct. If the path is in fact +// a bind mount from one part of a mount to another it will not be detected. +// mkdir /tmp/a /tmp/b; mount --bin /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") +// will return true. When in fact /tmp/b is a mount point. If this situation +// if of interest to you, don't use this function... +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return true, err + } + rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/"))) + if err != nil { + return true, err + } + // If the directory has a different device as parent, then it is a mountpoint. + if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { + return false, nil + } + + return true, nil +} + +// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. +// If pathname is not a device, log and return false with nil error. +// If open returns errno EBUSY, return true with nil error. +// If open returns nil, return false with nil error. +// Otherwise, return false with error +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return exclusiveOpenFailsOnDevice(pathname) +} + +// PathIsDevice uses FileInfo returned from os.Stat to check if path refers +// to a device. +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + pathType, err := mounter.GetFileType(pathname) + isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev + return isDevice, err +} + +func exclusiveOpenFailsOnDevice(pathname string) (bool, error) { + var isDevice bool + finfo, err := os.Stat(pathname) + if os.IsNotExist(err) { + isDevice = false + } + // err in call to os.Stat + if err != nil { + return false, fmt.Errorf( + "PathIsDevice failed for path %q: %v", + pathname, + err) + } + // path refers to a device + if finfo.Mode()&os.ModeDevice != 0 { + isDevice = true + } + + if !isDevice { + klog.Errorf("Path %q is not referring to a device.", pathname) + return false, nil + } + fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL, 0) + // If the device is in use, open will return an invalid fd. + // When this happens, it is expected that Close will fail and throw an error. + defer unix.Close(fd) + if errno == nil { + // device not in use + return false, nil + } else if errno == unix.EBUSY { + // device is in use + return true, nil + } + // error during call to Open + return false, errno +} + +//GetDeviceNameFromMount: given a mount point, find the device name from its global mount point +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginDir) +} + +// getDeviceNameFromMount find the device name from /proc/mounts in which +// the mount path reference should match the given plugin directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + klog.V(4).Infof("Directory %s is not mounted", mountPath) + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + basemountPath := path.Join(pluginDir, MountsInGlobalPDPath) + for _, ref := range refs { + if strings.HasPrefix(ref, basemountPath) { + volumeID, err := filepath.Rel(basemountPath, ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +func listProcMounts(mountFilePath string) ([]MountPoint, error) { + content, err := utilio.ConsistentRead(mountFilePath, maxListTries) + if err != nil { + return nil, err + } + return parseProcMounts(content) +} + +func parseProcMounts(content []byte) ([]MountPoint, error) { + out := []MountPoint{} + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) != expectedNumFieldsPerLine { + return nil, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line) + } + + mp := MountPoint{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: strings.Split(fields[3], ","), + } + + freq, err := strconv.Atoi(fields[4]) + if err != nil { + return nil, err + } + mp.Freq = freq + + pass, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, err + } + mp.Pass = pass + + out = append(out, mp) + } + return out, nil +} + +func (mounter *Mounter) MakeRShared(path string) error { + return doMakeRShared(path, procMountInfoPath) +} + +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +func (mounter *Mounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (mounter *Mounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return utilfile.FileExists(pathname) +} + +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// formatAndMount uses unix utils to format and mount the given disk +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + readOnly := false + for _, option := range options { + if option == "ro" { + readOnly = true + break + } + } + + options = append(options, "defaults") + + if !readOnly { + // Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw. + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + args := []string{"-a", source} + out, err := mounter.Exec.Run("fsck", args...) + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + klog.Warning("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + klog.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s.", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + klog.Infof("`fsck` error %s", string(out)) + } + } + } + + // Try to mount the disk + klog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target) + mountErr := mounter.Interface.Mount(source, target, fstype, options) + if mountErr != nil { + // Mount failed. This indicates either that the disk is unformatted or + // it contains an unexpected filesystem. + existingFormat, err := mounter.GetDiskFormat(source) + if err != nil { + return err + } + if existingFormat == "" { + if readOnly { + // Don't attempt to format if mounting as readonly, return an error to reflect this. + return errors.New("failed to mount unformatted volume as read only") + } + + // Disk is unformatted so format it. + args := []string{source} + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + } + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + _, err := mounter.Exec.Run("mkfs."+fstype, args...) + if err == nil { + // the disk has been formatted successfully try to mount it again. + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + return mounter.Interface.Mount(source, target, fstype, options) + } + klog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err) + return err + } else { + // Disk is already formatted and failed to mount + if len(fstype) == 0 || fstype == existingFormat { + // This is mount error + return mountErr + } else { + // Block device is formatted with unexpected filesystem, let the user know + return fmt.Errorf("failed to mount the volume as %q, it already contains %s. Mount error: %v", fstype, existingFormat, mountErr) + } + } + } + return mountErr +} + +// GetDiskFormat uses 'blkid' to see if the given disk is unformatted +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + dataOut, err := mounter.Exec.Run("blkid", args...) + output := string(dataOut) + klog.V(4).Infof("Output: %q, err: %v", output, err) + + if err != nil { + if exit, ok := err.(utilexec.ExitError); ok { + if exit.ExitStatus() == 2 { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + } + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return "", err + } + + var fstype, pttype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + cs := strings.Split(l, "=") + if len(cs) != 2 { + return "", fmt.Errorf("blkid returns invalid output: %s", output) + } + // TYPE is filesystem type, and PTTYPE is partition table type, according + // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. + if cs[0] == "TYPE" { + fstype = cs[1] + } else if cs[0] == "PTTYPE" { + pttype = cs[1] + } + } + + if len(pttype) > 0 { + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + // Returns a special non-empty string as filesystem type, then kubelet + // will not format it. + return "unknown data, probably partitions", nil + } + + return fstype, nil +} + +// isShared returns true, if given path is on a mount point that has shared +// mount propagation. +func isShared(mount string, mountInfoPath string) (bool, error) { + info, err := findMountInfo(mount, mountInfoPath) + if err != nil { + return false, err + } + + // parse optional parameters + for _, opt := range info.optionalFields { + if strings.HasPrefix(opt, "shared:") { + return true, nil + } + } + return false, nil +} + +// This represents a single line in /proc//mountinfo. +type mountInfo struct { + // Unique ID for the mount (maybe reused after umount). + id int + // The ID of the parent mount (or of self for the root of this mount namespace's mount tree). + parentID int + // The value of `st_dev` for files on this filesystem. + majorMinor string + // The pathname of the directory in the filesystem which forms the root of this mount. + root string + // Mount source, filesystem-specific information. e.g. device, tmpfs name. + source string + // Mount point, the pathname of the mount point. + mountPoint string + // Optional fields, zero or more fields of the form "tag[:value]". + optionalFields []string + // The filesystem type in the form "type[.subtype]". + fsType string + // Per-mount options. + mountOptions []string + // Per-superblock options. + superOptions []string +} + +// parseMountInfo parses /proc/xxx/mountinfo. +func parseMountInfo(filename string) ([]mountInfo, error) { + content, err := utilio.ConsistentRead(filename, maxListTries) + if err != nil { + return []mountInfo{}, err + } + contentStr := string(content) + infos := []mountInfo{} + + for _, line := range strings.Split(contentStr, "\n") { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + // See `man proc` for authoritative description of format of the file. + fields := strings.Fields(line) + if len(fields) < expectedAtLeastNumFieldsPerMountInfo { + return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line) + } + id, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, err + } + parentID, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err + } + info := mountInfo{ + id: id, + parentID: parentID, + majorMinor: fields[2], + root: fields[3], + mountPoint: fields[4], + mountOptions: strings.Split(fields[5], ","), + } + // All fields until "-" are "optional fields". + i := 6 + for ; i < len(fields) && fields[i] != "-"; i++ { + info.optionalFields = append(info.optionalFields, fields[i]) + } + // Parse the rest 3 fields. + i += 1 + if len(fields)-i < 3 { + return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i) + } + info.fsType = fields[i] + info.source = fields[i+1] + info.superOptions = strings.Split(fields[i+2], ",") + infos = append(infos, info) + } + return infos, nil +} + +func findMountInfo(path, mountInfoPath string) (mountInfo, error) { + infos, err := parseMountInfo(mountInfoPath) + if err != nil { + return mountInfo{}, err + } + + // process /proc/xxx/mountinfo in backward order and find the first mount + // point that is prefix of 'path' - that's the mount where path resides + var info *mountInfo + for i := len(infos) - 1; i >= 0; i-- { + if PathWithinBase(path, infos[i].mountPoint) { + info = &infos[i] + break + } + } + if info == nil { + return mountInfo{}, fmt.Errorf("cannot find mount point for %q", path) + } + return *info, nil +} + +// doMakeRShared is common implementation of MakeRShared on Linux. It checks if +// path is shared and bind-mounts it as rshared if needed. mountCmd and +// mountArgs are expected to contain mount-like command, doMakeRShared will add +// '--bind ' and '--make-rshared ' to mountArgs. +func doMakeRShared(path string, mountInfoFilename string) error { + shared, err := isShared(path, mountInfoFilename) + if err != nil { + return err + } + if shared { + klog.V(4).Infof("Directory %s is already on a shared mount", path) + return nil + } + + klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path) + // mount --bind /var/lib/kubelet /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil { + return fmt.Errorf("failed to bind-mount %s: %v", path, err) + } + + // mount --make-rshared /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil { + return fmt.Errorf("failed to make %s rshared: %v", path, err) + } + + return nil +} + +// getSELinuxSupport is common implementation of GetSELinuxSupport on Linux. +func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) { + info, err := findMountInfo(path, mountInfoFilename) + if err != nil { + return false, err + } + + // "seclabel" can be both in mount options and super options. + for _, opt := range info.superOptions { + if opt == "seclabel" { + return true, nil + } + } + for _, opt := range info.mountOptions { + if opt == "seclabel" { + return true, nil + } + } + return false, nil +} + +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + newHostPath, err = doBindSubPath(mounter, subPath) + + // There is no action when the container starts. Bind-mount will be cleaned + // when container stops by CleanSubPaths. + cleanupAction = nil + return newHostPath, cleanupAction, err +} + +// This implementation is shared between Linux and NsEnterMounter +func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) { + if !PathWithinBase(subpath.Path, subpath.VolumePath) { + return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath) + } + fd, err := doSafeOpen(subpath.Path, subpath.VolumePath) + if err != nil { + return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err) + } + return fd, nil +} + +// prepareSubpathTarget creates target for bind-mount of subpath. It returns +// "true" when the target already exists and something is mounted there. +// Given Subpath must have all paths with already resolved symlinks and with +// paths relevant to kubelet (when it runs in a container). +// This function is called also by NsEnterMounter. It works because +// /var/lib/kubelet is mounted from the host into the container with Kubelet as +// /var/lib/kubelet too. +func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) { + // Early check for already bind-mounted subpath. + bindPathTarget := getSubpathBindTarget(subpath) + notMount, err := IsNotMountPoint(mounter, bindPathTarget) + if err != nil { + if !os.IsNotExist(err) { + return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err) + } + // Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet. + notMount = true + } + if !notMount { + // It's already mounted + klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) + return true, bindPathTarget, nil + } + + // bindPathTarget is in /var/lib/kubelet and thus reachable without any + // translation even to containerized kubelet. + bindParent := filepath.Dir(bindPathTarget) + err = os.MkdirAll(bindParent, 0750) + if err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err) + } + + t, err := os.Lstat(subpath.Path) + if err != nil { + return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err) + } + + if t.Mode()&os.ModeDir > 0 { + if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err) + } + } else { + // "/bin/touch ". + // A file is enough for all possible targets (symlink, device, pipe, + // socket, ...), bind-mounting them into a file correctly changes type + // of the target file. + if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil { + return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err) + } + } + return false, bindPathTarget, nil +} + +func getSubpathBindTarget(subpath Subpath) string { + // containerName is DNS label, i.e. safe as a directory name. + return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex)) +} + +func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) { + // Linux, kubelet runs on the host: + // - safely open the subpath + // - bind-mount /proc//fd/ to subpath target + // User can't change /proc//fd/ to point to a bad place. + + // Evaluate all symlinks here once for all subsequent functions. + newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) + } + newPath, err := filepath.EvalSymlinks(subpath.Path) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) + } + klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath) + subpath.VolumePath = newVolumePath + subpath.Path = newPath + + fd, err := safeOpenSubPath(mounter, subpath) + if err != nil { + return "", err + } + defer syscall.Close(fd) + + alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) + if err != nil { + return "", err + } + if alreadyMounted { + return bindPathTarget, nil + } + + success := false + defer func() { + // Cleanup subpath on error + if !success { + klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { + klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + } + } + }() + + kubeletPid := os.Getpid() + mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) + + // Do the bind mount + options := []string{"bind"} + klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) + if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { + return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) + } + success = true + + klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + return bindPathTarget, nil +} + +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(mounter, podDir, volumeName) +} + +// This implementation is shared between Linux and NsEnterMounter +func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error { + // scan /var/lib/kubelet/pods//volume-subpaths//* + subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) + klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) + + containerDirs, err := ioutil.ReadDir(subPathDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("error reading %s: %s", subPathDir, err) + } + + for _, containerDir := range containerDirs { + if !containerDir.IsDir() { + klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) + continue + } + klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) + + // scan /var/lib/kubelet/pods//volume-subpaths///* + fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name()) + subPaths, err := ioutil.ReadDir(fullContainerDirPath) + if err != nil { + return fmt.Errorf("error reading %s: %s", fullContainerDirPath, err) + } + for _, subPath := range subPaths { + if err = doCleanSubPath(mounter, fullContainerDirPath, subPath.Name()); err != nil { + return err + } + } + // Whole container has been processed, remove its directory. + if err := os.Remove(fullContainerDirPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err) + } + klog.V(5).Infof("Removed %s", fullContainerDirPath) + } + // Whole pod volume subpaths have been cleaned up, remove its subpath directory. + if err := os.Remove(subPathDir); err != nil { + return fmt.Errorf("error deleting %s: %s", subPathDir, err) + } + klog.V(5).Infof("Removed %s", subPathDir) + + // Remove entire subpath directory if it's the last one + podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName) + if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) { + return fmt.Errorf("error deleting %s: %s", podSubPathDir, err) + } + klog.V(5).Infof("Removed %s", podSubPathDir) + return nil +} + +// doCleanSubPath tears down the single subpath bind mount +func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error { + // process /var/lib/kubelet/pods//volume-subpaths/// + klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) + fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) + notMnt, err := IsNotMountPoint(mounter, fullSubPath) + if err != nil { + return fmt.Errorf("error checking %s for mount: %s", fullSubPath, err) + } + // Unmount it + if !notMnt { + if err = mounter.Unmount(fullSubPath); err != nil { + return fmt.Errorf("error unmounting %s: %s", fullSubPath, err) + } + klog.V(5).Infof("Unmounted %s", fullSubPath) + } + // Remove it *non*-recursively, just in case there were some hiccups. + if err = os.Remove(fullSubPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullSubPath, err) + } + klog.V(5).Infof("Removed %s", fullSubPath) + return nil +} + +// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty +func cleanSubPath(mounter Interface, subpath Subpath) error { + containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) + + // Clean subdir bindmount + if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) { + return err + } + + // Recursively remove directories if empty + if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil { + return err + } + + return nil +} + +// removeEmptyDirs works backwards from endDir to baseDir and removes each directory +// if it is empty. It stops once it encounters a directory that has content +func removeEmptyDirs(baseDir, endDir string) error { + if !PathWithinBase(endDir, baseDir) { + return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) + } + + for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) { + s, err := os.Stat(curDir) + if err != nil { + if os.IsNotExist(err) { + klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) + continue + } + return fmt.Errorf("error stat %q: %v", curDir, err) + } + if !s.IsDir() { + return fmt.Errorf("path %q not a directory", curDir) + } + + err = os.Remove(curDir) + if os.IsExist(err) { + klog.V(5).Infof("Directory %q not empty, not removing", curDir) + break + } else if err != nil { + return fmt.Errorf("error removing directory %q: %v", curDir, err) + } + klog.V(5).Infof("Removed directory %q", curDir) + } + return nil +} + +func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + + return doSafeMakeDir(realFullPath, realBase, perm) +} + +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + if _, err := os.Stat(pathname); os.IsNotExist(err) { + return []string{}, nil + } else if err != nil { + return nil, err + } + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return searchMountPoints(realpath, procMountInfoPath) +} + +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + return getSELinuxSupport(pathname, procMountInfoPath) +} + +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return 0, err + } + return getFSGroup(realpath) +} + +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + return getMode(pathname) +} + +// This implementation is shared between Linux and NsEnterMounter +func getFSGroup(pathname string) (int64, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return int64(info.Sys().(*syscall.Stat_t).Gid), nil +} + +// This implementation is shared between Linux and NsEnterMounter +func getMode(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} + +// This implementation is shared between Linux and NsEnterMounter. Both pathname +// and base must be either already resolved symlinks or that will be resolved in +// kubelet's mount namespace (in case it runs containerized). +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", existingPath, err) + } + if !PathWithinBase(fullExistingPath, base) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + parentFD, err := doSafeOpen(fullExistingPath, base) + if err != nil { + return fmt.Errorf("cannot open directory %s: %s", existingPath, err) + } + childFD := -1 + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + } + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink. + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + err = syscall.Mkdirat(parentFD, currentPath, uint32(perm)) + if err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + // Dive into the created directory + childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0) + if err != nil { + return fmt.Errorf("cannot open %s: %s", currentPath, err) + } + // We can be sure that childFD is safe to use. It could be changed + // by user after Mkdirat() and before Openat(), however: + // - it could not be changed to symlink - we use nofollowFlags + // - it could be changed to a file (or device, pipe, socket, ...) + // but either subsequent Mkdirat() fails or we mount this file + // to user's container. Security is no violated in both cases + // and user either gets error or the file that it can already access. + + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + parentFD = childFD + childFD = -1 + } + + // Everything was created. mkdirat(..., perm) above was affected by current + // umask and we must apply the right permissions to the last directory + // (that's the one that will be available to the container as subpath) + // so user can read/write it. This is the behavior of previous code. + // TODO: chmod all created directories, not just the last one. + // parentFD is the last created directory. + + // Translate perm (os.FileMode) to uint32 that fchmod() expects + kernelPerm := uint32(perm & os.ModePerm) + if perm&os.ModeSetgid > 0 { + kernelPerm |= syscall.S_ISGID + } + if perm&os.ModeSetuid > 0 { + kernelPerm |= syscall.S_ISUID + } + if perm&os.ModeSticky > 0 { + kernelPerm |= syscall.S_ISVTX + } + if err = syscall.Fchmod(parentFD, kernelPerm); err != nil { + return fmt.Errorf("chmod %q failed: %s", currentPath, err) + } + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + dirs := strings.Split(rel, string(filepath.Separator)) + + // Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks. + // This should be faster than looping through all dirs and calling os.Stat() + // on each of them, as the symlinks are resolved only once with OpenAt(). + currentPath := base + fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0) + if err != nil { + return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err) + } + defer func() { + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + }() + for i, dir := range dirs { + // Using O_PATH here will prevent hangs in case user replaces directory with + // fifo + childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0) + if err != nil { + if os.IsNotExist(err) { + return currentPath, dirs[i:], nil + } + return base, nil, err + } + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + fd = childFD + currentPath = filepath.Join(currentPath, dir) + } + return pathname, []string{}, nil +} + +// This implementation is shared between Linux and NsEnterMounter +// Open path and return its fd. +// Symlinks are disallowed (pathname must already resolve symlinks), +// and the path must be within the base directory. +func doSafeOpen(pathname string, base string) (int, error) { + pathname = filepath.Clean(pathname) + base = filepath.Clean(base) + + // Calculate segments to follow + subpath, err := filepath.Rel(base, pathname) + if err != nil { + return -1, err + } + segments := strings.Split(subpath, string(filepath.Separator)) + + // Assumption: base is the only directory that we have under control. + // Base dir is not allowed to be a symlink. + parentFD, err := syscall.Open(base, nofollowFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open directory %s: %s", base, err) + } + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) + } + } + }() + + childFD := -1 + defer func() { + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := base + + // Follow the segments one by one using openat() to make + // sure the user cannot change already existing directories into symlinks. + for _, seg := range segments { + currentPath = filepath.Join(currentPath, seg) + if !PathWithinBase(currentPath, base) { + return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) + } + + klog.V(5).Infof("Opening path %s", currentPath) + childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open %s: %s", currentPath, err) + } + + var deviceStat unix.Stat_t + err := unix.Fstat(childFD, &deviceStat) + if err != nil { + return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err) + } + fileFmt := deviceStat.Mode & syscall.S_IFMT + if fileFmt == syscall.S_IFLNK { + return -1, fmt.Errorf("Unexpected symlink found %s", currentPath) + } + + // Close parentFD + if err = syscall.Close(parentFD); err != nil { + return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err) + } + // Set child to new parent + parentFD = childFD + childFD = -1 + } + + // We made it to the end, return this fd, don't close it + finalFD := parentFD + parentFD = -1 + + return finalFD, nil +} + +// searchMountPoints finds all mount references to the source, returns a list of +// mountpoints. +// This function assumes source cannot be device. +// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, +// it's possible to mount a non-root path of a filesystem, so we need to use +// root path and major:minor to represent mount source uniquely. +// This implementation is shared between Linux and NsEnterMounter +func searchMountPoints(hostSource, mountInfoPath string) ([]string, error) { + mis, err := parseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + + mountID := 0 + rootPath := "" + majorMinor := "" + + // Finding the underlying root path and major:minor if possible. + // We need search in backward order because it's possible for later mounts + // to overlap earlier mounts. + for i := len(mis) - 1; i >= 0; i-- { + if hostSource == mis[i].mountPoint || PathWithinBase(hostSource, mis[i].mountPoint) { + // If it's a mount point or path under a mount point. + mountID = mis[i].id + rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint)) + majorMinor = mis[i].majorMinor + break + } + } + + if rootPath == "" || majorMinor == "" { + return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) + } + + var refs []string + for i := range mis { + if mis[i].id == mountID { + // Ignore mount entry for mount source itself. + continue + } + if mis[i].root == rootPath && mis[i].majorMinor == majorMinor { + refs = append(refs, mis[i].mountPoint) + } + } + + return refs, nil +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_unsupported.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_unsupported.go new file mode 100644 index 00000000000..cd6f1a28f28 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_unsupported.go @@ -0,0 +1,139 @@ +// +build !linux,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" + "os" +) + +type Mounter struct { + mounterPath string +} + +var unsupportedErr = errors.New("util/mount on this platform is not supported") + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return unsupportedErr +} + +func (mounter *Mounter) Unmount(target string) error { + return unsupportedErr +} + +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, unsupportedErr +} + +func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { + return (mp.Path == dir) +} + +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return IsNotMountPoint(mounter, dir) +} + +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + return true, unsupportedErr +} + +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return "", unsupportedErr +} + +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + return "", unsupportedErr +} + +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return false, unsupportedErr +} + +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + return true, unsupportedErr +} + +func (mounter *Mounter) MakeRShared(path string) error { + return unsupportedErr +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + return mounter.Interface.Mount(source, target, fstype, options) +} + +func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { + return true, unsupportedErr +} + +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return FileType("fake"), unsupportedErr +} + +func (mounter *Mounter) MakeDir(pathname string) error { + return unsupportedErr +} + +func (mounter *Mounter) MakeFile(pathname string) error { + return unsupportedErr +} + +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return true, errors.New("not implemented") +} + +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return "", unsupportedErr +} + +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, unsupportedErr +} + +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return unsupportedErr +} + +func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return unsupportedErr +} + +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} + +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + return false, errors.New("not implemented") +} + +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + return 0, errors.New("not implemented") +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_windows.go b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_windows.go new file mode 100644 index 00000000000..04a808df64d --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/openstack/util/mount/mount_windows.go @@ -0,0 +1,611 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + + "k8s.io/klog" + + utilfile "k8s.io/cloud-provider-openstack/pkg/util/file" +) + +// Mounter provides the default implementation of mount.Interface +// for the windows platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount : mounts source to target as NTFS with given options. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + target = normalizeWindowsPath(target) + + if source == "tmpfs" { + klog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options) + return os.MkdirAll(target, 0755) + } + + parentDir := filepath.Dir(target) + if err := os.MkdirAll(parentDir, 0755); err != nil { + return err + } + + klog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + options, source, target, fstype) + bindSource := "" + + // tell it's going to mount azure disk or azure file according to options + if bind, _, _ := isBind(options); bind { + // mount azure disk + bindSource = normalizeWindowsPath(source) + } else { + if len(options) < 2 { + klog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", + options, len(options), source, target) + return nil + } + + // currently only cifs mount is supported + if strings.ToLower(fstype) != "cifs" { + return fmt.Errorf("azureMount: only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options) + } + + bindSource = source + + // use PowerShell Environment Variables to store user input string to prevent command line injection + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 + cmdLine := fmt.Sprintf(`$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`) + + cmd := exec.Command("powershell", "/c", cmdLine) + cmd.Env = append(os.Environ(), + fmt.Sprintf("smbuser=%s", options[0]), + fmt.Sprintf("smbpassword=%s", options[1]), + fmt.Sprintf("smbremotepath=%s", source)) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("azureMount: SmbGlobalMapping failed: %v, only SMB mount is supported now, output: %q", err, string(output)) + } + } + + if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil { + klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) + return err + } + + return nil +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("azureMount: Unmount target (%q)", target) + target = normalizeWindowsPath(target) + if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { + klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// List returns a list of all mounted filesystems. todo +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, nil +} + +// IsMountPointMatch determines if the mountpoint matches the dir +func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { + return mp.Path == dir +} + +// IsNotMountPoint determines if a directory is a mountpoint. +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return IsNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Lstat(file) + if err != nil { + return true, err + } + // If current file is a symlink, then it is a mountpoint. + if stat.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(file) + if err != nil { + return true, fmt.Errorf("readlink error: %v", err) + } + exists, err := mounter.ExistsPath(target) + if err != nil { + return true, err + } + return !exists, nil + } + + return true, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginDir) +} + +// getDeviceNameFromMount find the device(drive) name in which +// the mount path reference should match the given plugin directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + basemountPath := normalizeWindowsPath(path.Join(pluginDir, MountsInGlobalPDPath)) + for _, ref := range refs { + if strings.Contains(ref, basemountPath) { + volumeID, err := filepath.Rel(normalizeWindowsPath(basemountPath), ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +// DeviceOpened determines if the device is in use elsewhere +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return false, nil +} + +// PathIsDevice determines if a path is a device. +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + return false, nil +} + +// MakeRShared checks that given path is on a mount with 'rshared' mount +// propagation. Empty implementation here. +func (mounter *Mounter) MakeRShared(path string) error { + return nil +} + +// GetFileType checks for sockets/block/character devices +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +// MakeFile creates a new directory +func (mounter *Mounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// MakeFile creates an empty file +func (mounter *Mounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// ExistsPath checks whether the path exists +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return utilfile.FileExists(pathname) +} + +// EvalHostSymlinks returns the path name after evaluating symlinks +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// check whether hostPath is within volume path +// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started +func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(hostPath) == 0 { + return []uintptr{}, nil + } + + finalSubPath, err := filepath.EvalSymlinks(hostPath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err) + } + finalVolumePath, err := filepath.EvalSymlinks(volumePath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err) + } + + return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath) +} + +// lock all intermediate subPath directories and check they are all within volumePath +// volumePath & subPath should not contain any symlink, otherwise it will return error +func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(subPath) == 0 { + return []uintptr{}, nil + } + + // get relative path to volumePath + relSubPath, err := filepath.Rel(volumePath, subPath) + if err != nil { + return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err) + } + if startsWithBackstep(relSubPath) { + return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath) + } + + if relSubPath == "." { + // volumePath and subPath are equal + return []uintptr{}, nil + } + + fileHandles := []uintptr{} + var errorResult error + + currentFullPath := volumePath + dirs := strings.Split(relSubPath, string(os.PathSeparator)) + for _, dir := range dirs { + // lock intermediate subPath directory first + currentFullPath = filepath.Join(currentFullPath, dir) + handle, err := lockPath(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err) + break + } + fileHandles = append(fileHandles, handle) + + // make sure intermediate subPath directory does not contain symlink any more + stat, err := os.Lstat(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err) + break + } + if stat.Mode()&os.ModeSymlink != 0 { + errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath) + break + } + + if !PathWithinBase(currentFullPath, volumePath) { + errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath) + break + } + } + + return fileHandles, errorResult +} + +// unlockPath unlock directories +func unlockPath(fileHandles []uintptr) { + if fileHandles != nil { + for _, handle := range fileHandles { + syscall.CloseHandle(syscall.Handle(handle)) + } + } +} + +// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path +func lockPath(path string) (uintptr, error) { + if len(path) == 0 { + return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return uintptr(syscall.InvalidHandle), err + } + access := uint32(syscall.GENERIC_READ) + sharemode := uint32(syscall.FILE_SHARE_READ) + createmode := uint32(syscall.OPEN_EXISTING) + flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT) + fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0) + return uintptr(fd), err +} + +// Lock all directories in subPath and check they're not symlinks. +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path) + + // Unlock the directories when the container starts + cleanupAction = func() { + unlockPath(handles) + } + return subPath.Path, cleanupAction, err +} + +// No bind-mounts for subpaths are necessary on Windows +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return nil +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + // Try to mount the disk + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + + if err := ValidateDiskNumber(source); err != nil { + klog.Errorf("diskMount: formatAndMount failed, err: %v", err) + return err + } + + if len(fstype) == 0 { + // Use 'NTFS' as the default + fstype = "NTFS" + } + + // format disk if it is unformatted(raw) + cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ + " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) + if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil { + return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) + } + klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) + + driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec) + if err != nil { + return err + } + driverPath := driveLetter + ":" + target = normalizeWindowsPath(target) + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) + if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil { + klog.Errorf("mklink failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +func normalizeWindowsPath(path string) string { + normalizedPath := strings.Replace(path, "/", "\\", -1) + if strings.HasPrefix(normalizedPath, "\\") { + normalizedPath = "c:" + normalizedPath + } + return normalizedPath +} + +// ValidateDiskNumber : disk number should be a number in [0, 99] +func ValidateDiskNumber(disk string) error { + diskNum, err := strconv.Atoi(disk) + if err != nil { + return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err) + } + + if diskNum < 0 || diskNum > 99 { + return fmt.Errorf("disk number out of range: %q", disk) + } + + return nil +} + +// Get drive letter according to windows disk number +func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) { + cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum) + output, err := exec.Run("powershell", "/c", cmd) + if err != nil { + return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output)) + } + if len(string(output)) < 1 { + return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty") + } + return string(output)[:1], nil +} + +// getAllParentLinks walks all symbolic links and return all the parent targets recursively +func getAllParentLinks(path string) ([]string, error) { + const maxIter = 255 + links := []string{} + for { + links = append(links, path) + if len(links) > maxIter { + return links, fmt.Errorf("unexpected length of parent links: %v", links) + } + + fi, err := os.Lstat(path) + if err != nil { + return links, fmt.Errorf("Lstat: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + break + } + + path, err = os.Readlink(path) + if err != nil { + return links, fmt.Errorf("Readlink error: %v", err) + } + } + + return links, nil +} + +// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) { + return []string{}, nil + } else if err != nil { + return nil, err + } + return []string{pathname}, nil +} + +// Note that on windows, it always returns 0. We actually don't set FSGroup on +// windows platform, see SetVolumeOwnership implementation. +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return 0, nil +} + +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + // Windows does not support SELinux. + return false, nil +} + +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} + +// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. +func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + return doSafeMakeDir(realFullPath, realBase, perm) +} + +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + if len(toCreate) == 0 { + return nil + } + + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening existing directory %s: %s", existingPath, err) + } + fullBasePath, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("cannot read link %s: %s", base, err) + } + if !PathWithinBase(fullExistingPath, fullBasePath) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + // lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom) + fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath) + defer unlockPath(fileHandles) + if err != nil { + return err + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink by lock that directory immediately + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + if err := os.Mkdir(currentPath, perm); err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + handle, err := lockPath(currentPath) + if err != nil { + return fmt.Errorf("cannot lock path %s: %s", currentPath, err) + } + defer syscall.CloseHandle(syscall.Handle(handle)) + // make sure newly created directory does not contain symlink after lock + stat, err := os.Lstat(currentPath) + if err != nil { + return fmt.Errorf("Lstat(%q) error: %v", currentPath, err) + } + if stat.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath) + } + } + + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + + if startsWithBackstep(rel) { + return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base) + } + + if rel == "." { + // base and pathname are equal + return pathname, []string{}, nil + } + + dirs := strings.Split(rel, string(filepath.Separator)) + + parent := base + currentPath := base + for i, dir := range dirs { + parent = currentPath + currentPath = filepath.Join(parent, dir) + if _, err := os.Lstat(currentPath); err != nil { + if os.IsNotExist(err) { + return parent, dirs[i:], nil + } + return base, nil, err + } + } + + return pathname, []string{}, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8363ac01ecc..fc897b8974b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1831,6 +1831,7 @@ k8s.io/legacy-cloud-providers/aws k8s.io/legacy-cloud-providers/azure k8s.io/legacy-cloud-providers/azure/auth k8s.io/legacy-cloud-providers/gce +k8s.io/legacy-cloud-providers/openstack k8s.io/legacy-cloud-providers/vsphere k8s.io/legacy-cloud-providers/vsphere/testing k8s.io/legacy-cloud-providers/vsphere/vclib