From cb5eb25ec197e6bb97c4044effa5e2489b889651 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Tue, 22 May 2018 12:56:26 +0200 Subject: [PATCH] Nsenter unit tests --- pkg/util/mount/BUILD | 1 + pkg/util/mount/nsenter_mount_test.go | 560 ++++++++++++++++++++++++ pkg/util/nsenter/BUILD | 19 +- pkg/util/nsenter/nsenter.go | 121 ++++- pkg/util/nsenter/nsenter_test.go | 311 +++++++++++++ pkg/util/nsenter/nsenter_unsupported.go | 8 +- 6 files changed, 995 insertions(+), 25 deletions(-) create mode 100644 pkg/util/nsenter/nsenter_test.go diff --git a/pkg/util/mount/BUILD b/pkg/util/mount/BUILD index 40778e7c73b..6f2df9acae8 100644 --- a/pkg/util/mount/BUILD +++ b/pkg/util/mount/BUILD @@ -133,6 +133,7 @@ go_test( "//vendor/k8s.io/utils/exec/testing:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/util/nsenter:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", diff --git a/pkg/util/mount/nsenter_mount_test.go b/pkg/util/mount/nsenter_mount_test.go index 2edb6febf97..c541a4cdf74 100644 --- a/pkg/util/mount/nsenter_mount_test.go +++ b/pkg/util/mount/nsenter_mount_test.go @@ -26,6 +26,7 @@ import ( "testing" "golang.org/x/sys/unix" + "k8s.io/kubernetes/pkg/util/nsenter" ) func TestParseFindMnt(t *testing.T) { @@ -147,3 +148,562 @@ func TestCheckDeviceInode(t *testing.T) { } } } + +func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) { + rootfsPath = filepath.Join(tmpdir, "rootfs") + if err := os.Mkdir(rootfsPath, 0755); err != nil { + return nil, "", "", err + } + ne, err := nsenter.NewFakeNsenter(rootfsPath) + if err != nil { + return nil, "", "", err + } + + varlibPath = filepath.Join(tmpdir, "/var/lib/kubelet") + if err := os.MkdirAll(varlibPath, 0755); err != nil { + return nil, "", "", err + } + + return NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, nil +} + +func TestNsenterExistsFile(t *testing.T) { + tests := []struct { + name string + prepare func(base, rootfs string) (string, error) + expectedOutput bool + expectError bool + }{ + { + name: "simple existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/file + path := filepath.Join(base, "file") + if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { + return "", err + } + // In kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, path, 0644); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: true, + }, + { + name: "simple non-existing file", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "file") + return path, nil + }, + expectedOutput: false, + }, + { + name: "simple non-accessible file", + prepare: func(base, rootfs string) (string, error) { + // On the host: + // create /base/dir/file, then make the dir inaccessible + dir := filepath.Join(base, "dir") + if err := os.MkdirAll(dir, 0755); err != nil { + return "", err + } + path := filepath.Join(dir, "file") + if err := ioutil.WriteFile(path, []byte{}, 0); err != nil { + return "", err + } + if err := os.Chmod(dir, 0644); err != nil { + return "", err + } + + // In kubelet: do the same with /rootfs/base/dir/file + rootfsPath, err := writeRootfsFile(rootfs, path, 0777) + if err != nil { + return "", err + } + rootfsDir := filepath.Dir(rootfsPath) + if err := os.Chmod(rootfsDir, 0644); err != nil { + return "", err + } + + return path, nil + }, + expectedOutput: false, + expectError: true, + }, + { + name: "relative symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink("file", path); err != nil { + return "", err + } + // In kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: true, + }, + { + name: "absolute symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> /base/file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink(file, path); err != nil { + return "", err + } + // In kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { + return "", err + } + + return path, nil + }, + expectedOutput: true, + }, + { + name: "relative symlink to non-existing file", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "link") + if err := os.Symlink("file", path); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: false, + }, + { + name: "absolute symlink to non-existing file", + prepare: func(base, rootfs string) (string, error) { + file := filepath.Join(base, "file") + path := filepath.Join(base, "link") + if err := os.Symlink(file, path); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: false, + }, + { + name: "symlink loop", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "link") + if err := os.Symlink(path, path); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: false, + // TODO: realpath -m is not able to detect symlink loop. Should we care? + expectError: false, + }, + } + + for _, test := range tests { + tmpdir, err := ioutil.TempDir("", "nsenter-exists-file") + if err != nil { + t.Error(err) + continue + } + defer os.RemoveAll(tmpdir) + + testBase := filepath.Join(tmpdir, "base") + if err := os.Mkdir(testBase, 0755); err != nil { + t.Error(err) + continue + } + + mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t) + if err != nil { + t.Error(err) + continue + } + + path, err := test.prepare(testBase, rootfs) + if err != nil { + t.Error(err) + continue + } + + out, err := mounter.ExistsPath(path) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("Test %q: expected error, got none", test.name) + } + + if out != test.expectedOutput { + t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedOutput, out) + } + } +} + +func TestNsenterGetMode(t *testing.T) { + tests := []struct { + name string + prepare func(base, rootfs string) (string, error) + expectedMode os.FileMode + expectError bool + }{ + { + name: "simple file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/file + path := filepath.Join(base, "file") + if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { + return "", err + } + + // Prepare a different file as /rootfs/base/file (="the host + // visible from container") to check that NsEnterMounter calls + // stat on this file and not on /base/file. + // Visible from kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, path, 0777); err != nil { + return "", err + } + + return path, nil + }, + expectedMode: 0777, + }, + { + name: "non-existing file", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "file") + return path, nil + }, + expectedMode: 0, + expectError: true, + }, + { + name: "absolute symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> /base/file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0644); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink(file, path); err != nil { + return "", err + } + + // Visible from kubelet: + // /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0747); err != nil { + return "", err + } + + return path, nil + }, + expectedMode: 0747, + }, + { + name: "relative symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0741); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink("file", path); err != nil { + return "", err + } + + // Visible from kubelet: + // /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0647); err != nil { + return "", err + } + + return path, nil + }, + expectedMode: 0647, + }, + } + + for _, test := range tests { + tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") + if err != nil { + t.Error(err) + continue + } + defer os.RemoveAll(tmpdir) + + testBase := filepath.Join(tmpdir, "base") + if err := os.Mkdir(testBase, 0755); err != nil { + t.Error(err) + continue + } + + mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t) + if err != nil { + t.Error(err) + continue + } + + path, err := test.prepare(testBase, rootfs) + if err != nil { + t.Error(err) + continue + } + + mode, err := mounter.GetMode(path) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("Test %q: expected error, got none", test.name) + } + + if mode != test.expectedMode { + t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedMode, mode) + } + } +} + +func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) { + fullPath := filepath.Join(rootfs, path) + dir := filepath.Dir(fullPath) + if err := os.MkdirAll(dir, 0755); err != nil { + return "", err + } + if err := ioutil.WriteFile(fullPath, []byte{}, mode); err != nil { + return "", err + } + // Use chmod, io.WriteFile is affected by umask + if err := os.Chmod(fullPath, mode); err != nil { + return "", err + } + return fullPath, nil +} + +func TestNsenterSafeMakeDir(t *testing.T) { + tests := []struct { + name string + prepare func(base, rootfs, varlib string) (expectedDir string, err error) + subdir string + expectError bool + // If true, "base" directory for SafeMakeDir will be /var/lib/kubelet + baseIsVarLib bool + }{ + { + name: "simple directory", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // expected to be created in /roots/ + expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure") + return expectedDir, nil + }, + }, + { + name: "simple existing directory", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: directory exists + hostPath := filepath.Join(base, "some/subdirectory/structure") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + // In rootfs: directory exists + kubeletPath := filepath.Join(rootfs, hostPath) + if err := os.MkdirAll(kubeletPath, 0755); err != nil { + return "", err + } + // expected to be created in /roots/ + expectedDir = kubeletPath + return expectedDir, nil + }, + }, + { + name: "absolute symlink into safe place", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /base/other/subdirectory exists, /base/some is link to /base/other + hostPath := filepath.Join(base, "other/subdirectory") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + somePath := filepath.Join(base, "some") + otherPath := filepath.Join(base, "other") + if err := os.Symlink(otherPath, somePath); err != nil { + return "", err + } + + // In rootfs: /base/other/subdirectory exists + kubeletPath := filepath.Join(rootfs, hostPath) + if err := os.MkdirAll(kubeletPath, 0755); err != nil { + return "", err + } + // expected 'structure' to be created + expectedDir = filepath.Join(rootfs, hostPath, "structure") + return expectedDir, nil + }, + }, + { + name: "relative symlink into safe place", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /base/other/subdirectory exists, /base/some is link to other + hostPath := filepath.Join(base, "other/subdirectory") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + somePath := filepath.Join(base, "some") + if err := os.Symlink("other", somePath); err != nil { + return "", err + } + + // In rootfs: /base/other/subdirectory exists + kubeletPath := filepath.Join(rootfs, hostPath) + if err := os.MkdirAll(kubeletPath, 0755); err != nil { + return "", err + } + // expected 'structure' to be created + expectedDir = filepath.Join(rootfs, hostPath, "structure") + return expectedDir, nil + }, + }, + { + name: "symlink into unsafe place", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /base/some is link to /bin/other + somePath := filepath.Join(base, "some") + if err := os.Symlink("/bin", somePath); err != nil { + return "", err + } + return "", nil + }, + expectError: true, + }, + { + name: "simple directory in /var/lib/kubelet", + // evaluated in varlib + subdir: "some/subdirectory/structure", + baseIsVarLib: true, + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // expected to be created in /base/var/lib/kubelet, not in /rootfs! + expectedDir = filepath.Join(varlib, "some/subdirectory/structure") + return expectedDir, nil + }, + }, + { + name: "safe symlink in /var/lib/kubelet", + // evaluated in varlib + subdir: "some/subdirectory/structure", + baseIsVarLib: true, + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other + hostPath := filepath.Join(varlib, "other/subdirectory") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + somePath := filepath.Join(varlib, "some") + if err := os.Symlink("other", somePath); err != nil { + return "", err + } + + // expected to be created in /base/var/lib/kubelet, not in /rootfs! + expectedDir = filepath.Join(varlib, "other/subdirectory/structure") + return expectedDir, nil + }, + }, + { + name: "unsafe symlink in /var/lib/kubelet", + // evaluated in varlib + subdir: "some/subdirectory/structure", + baseIsVarLib: true, + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /varlib/some is link to /bin + somePath := filepath.Join(varlib, "some") + if err := os.Symlink("/bin", somePath); err != nil { + return "", err + } + + return "", nil + }, + expectError: true, + }, + } + for _, test := range tests { + tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") + if err != nil { + t.Error(err) + continue + } + defer os.RemoveAll(tmpdir) + + mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t) + if err != nil { + t.Error(err) + continue + } + // Prepare base directory for the test + testBase := filepath.Join(tmpdir, "base") + if err := os.Mkdir(testBase, 0755); err != nil { + t.Error(err) + continue + } + // Prepare base directory also in /rootfs + rootfsBase := filepath.Join(rootfs, testBase) + if err := os.MkdirAll(rootfsBase, 0755); err != nil { + t.Error(err) + continue + } + + expectedDir := "" + if test.prepare != nil { + expectedDir, err = test.prepare(testBase, rootfs, varlib) + if err != nil { + t.Error(err) + continue + } + } + + if test.baseIsVarLib { + // use /var/lib/kubelet as the test base so we can test creating + // subdirs there directly in /var/lib/kubenet and not in + // /rootfs/var/lib/kubelet + testBase = varlib + } + + err = mounter.SafeMakeDir(test.subdir, testBase, 0755) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if test.expectError { + if err == nil { + t.Errorf("Test %q: expected error, got none", test.name) + } else { + if !strings.Contains(err.Error(), "is outside of allowed base") { + t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err) + } + } + } + + if expectedDir != "" { + _, err := os.Stat(expectedDir) + if err != nil { + t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err) + } + } + } +} diff --git a/pkg/util/nsenter/BUILD b/pkg/util/nsenter/BUILD index 988fef01b59..286b8882773 100644 --- a/pkg/util/nsenter/BUILD +++ b/pkg/util/nsenter/BUILD @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -92,3 +92,20 @@ filegroup( tags = ["automanaged"], visibility = ["//visibility:public"], ) + +go_test( + name = "go_default_test", + srcs = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "nsenter_test.go", + ], + "//conditions:default": [], + }), + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "//conditions:default": [], + }), +) diff --git a/pkg/util/nsenter/nsenter.go b/pkg/util/nsenter/nsenter.go index 477950476b1..e928a57ac9f 100644 --- a/pkg/util/nsenter/nsenter.go +++ b/pkg/util/nsenter/nsenter.go @@ -19,6 +19,8 @@ limitations under the License. package nsenter import ( + "context" + "errors" "fmt" "os" "path/filepath" @@ -30,9 +32,11 @@ import ( ) const ( - hostRootFsPath = "/rootfs" - // hostProcMountNsPath is the default mount namespace for rootfs - hostProcMountNsPath = "/rootfs/proc/1/ns/mnt" + // DefaultHostRootFsPath is path to host's filesystem mounted into container + // with kubelet. + DefaultHostRootFsPath = "/rootfs" + // mountNsPath is the default mount namespace of the host + mountNsPath = "/proc/1/ns/mnt" // nsenterPath is the default nsenter command nsenterPath = "nsenter" ) @@ -65,30 +69,46 @@ const ( type Nsenter struct { // a map of commands to their paths on the host filesystem paths map[string]string + + // Path to the host filesystem, typically "/rootfs". Used only for testing. + hostRootFsPath string + + // Exec implementation, used only for testing + executor exec.Interface } // NewNsenter constructs a new instance of Nsenter -func NewNsenter() (*Nsenter, error) { +func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) { ne := &Nsenter{ - paths: map[string]string{ - "mount": "", - "findmnt": "", - "umount": "", - "systemd-run": "", - "stat": "", - "touch": "", - "mkdir": "", - "ls": "", - "sh": "", - "chmod": "", - }, + hostRootFsPath: hostRootFsPath, + executor: executor, + } + if err := ne.initPaths(); err != nil { + return nil, err + } + return ne, nil +} + +func (ne *Nsenter) initPaths() error { + ne.paths = map[string]string{} + binaries := []string{ + "mount", + "findmnt", + "umount", + "systemd-run", + "stat", + "touch", + "mkdir", + "sh", + "chmod", + "realpath", } // search for the required commands in other locations besides /usr/bin - for binary := range ne.paths { + for _, binary := range binaries { // check for binary under the following directories for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} { binPath := filepath.Join(path, binary) - if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil { + if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil { continue } ne.paths[binary] = binPath @@ -96,19 +116,19 @@ func NewNsenter() (*Nsenter, error) { } // systemd-run is optional, bailout if we don't find any of the other binaries if ne.paths[binary] == "" && binary != "systemd-run" { - return nil, fmt.Errorf("unable to find %v", binary) + return fmt.Errorf("unable to find %v", binary) } } - return ne, nil + return nil } // Exec executes nsenter commands in hostProcMountNsPath mount namespace func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd { + hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath) fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"}, append([]string{ne.AbsHostPath(cmd)}, args...)...) glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) - exec := exec.New() - return exec.Command(nsenterPath, fullArgs...) + return ne.executor.Command(nsenterPath, fullArgs...) } // AbsHostPath returns the absolute runnable path for a specified command @@ -136,6 +156,9 @@ func (ne *Nsenter) SupportsSystemd() (string, bool) { // non/existing/directory does not exist // -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns // /mnt/foo/non/existing/directory. +// +// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false! +// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo. func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) { var args []string if mustExist { @@ -157,5 +180,57 @@ func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) // kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks // before calling this function func (ne *Nsenter) KubeletPath(pathname string) string { - return filepath.Join(hostRootFsPath, pathname) + return filepath.Join(ne.hostRootFsPath, pathname) } + +// NewFakeNsenter returns a Nsenter that does not run "nsenter --mount=... --", +// but runs everything in the same mount namespace as the unit test binary. +// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /. +// This fake Nsenter is enough for most operations, e.g. to resolve symlinks, +// but it's not enough to call /bin/mount - unit tests don't run as root. +func NewFakeNsenter(rootfsPath string) (*Nsenter, error) { + executor := &fakeExec{ + rootfsPath: rootfsPath, + } + // prepare /rootfs/bin, usr/bin and usr/sbin + bin := filepath.Join(rootfsPath, "bin") + if err := os.Symlink("/bin", bin); err != nil { + return nil, err + } + + usr := filepath.Join(rootfsPath, "usr") + if err := os.Mkdir(usr, 0755); err != nil { + return nil, err + } + usrbin := filepath.Join(usr, "bin") + if err := os.Symlink("/usr/bin", usrbin); err != nil { + return nil, err + } + usrsbin := filepath.Join(usr, "sbin") + if err := os.Symlink("/usr/sbin", usrsbin); err != nil { + return nil, err + } + + return NewNsenter(rootfsPath, executor) +} + +type fakeExec struct { + rootfsPath string +} + +func (f fakeExec) Command(cmd string, args ...string) exec.Cmd { + // This will intentionaly panic if Nsenter does not provide enough arguments. + realCmd := args[2] + realArgs := args[3:] + return exec.New().Command(realCmd, realArgs...) +} + +func (fakeExec) LookPath(file string) (string, error) { + return "", errors.New("not implemented") +} + +func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { + return nil +} + +var _ exec.Interface = fakeExec{} diff --git a/pkg/util/nsenter/nsenter_test.go b/pkg/util/nsenter/nsenter_test.go new file mode 100644 index 00000000000..3158a55bbec --- /dev/null +++ b/pkg/util/nsenter/nsenter_test.go @@ -0,0 +1,311 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nsenter + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "k8s.io/utils/exec" +) + +func TestExec(t *testing.T) { + tests := []struct { + name string + command string + args []string + expectedOutput string + expectError bool + }{ + { + name: "simple command", + command: "echo", + args: []string{"hello", "world"}, + expectedOutput: "hello world\n", + }, + { + name: "nozero exit code", + command: "false", + expectError: true, + }, + } + + executor := fakeExec{ + rootfsPath: "/rootfs", + } + for _, test := range tests { + ns := Nsenter{ + hostRootFsPath: "/rootfs", + executor: executor, + } + cmd := ns.Exec(test.command, test.args) + outBytes, err := cmd.CombinedOutput() + out := string(outBytes) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("Test %q: expected error, got none", test.name) + } + if test.expectedOutput != out { + t.Errorf("test %q: expected output %q, got %q", test.name, test.expectedOutput, out) + } + } +} + +func TestKubeletPath(t *testing.T) { + tests := []struct { + rootfs string + hostpath string + expectedKubeletPath string + }{ + { + // simple join + "/rootfs", + "/some/path", + "/rootfs/some/path", + }, + { + // squash slashes + "/rootfs/", + "//some/path", + "/rootfs/some/path", + }, + } + + for _, test := range tests { + ns := Nsenter{ + hostRootFsPath: test.rootfs, + } + out := ns.KubeletPath(test.hostpath) + if out != test.expectedKubeletPath { + t.Errorf("Expected path %q, got %q", test.expectedKubeletPath, out) + } + + } +} + +func TestEvalSymlinks(t *testing.T) { + tests := []struct { + name string + mustExist bool + prepare func(tmpdir string) (src string, expectedDst string, err error) + expectError bool + }{ + { + name: "simple file /src", + mustExist: true, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + src = filepath.Join(tmpdir, "src") + err = ioutil.WriteFile(src, []byte{}, 0644) + return src, src, err + }, + }, + { + name: "non-existing file /src", + mustExist: true, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + src = filepath.Join(tmpdir, "src") + return src, "", nil + }, + expectError: true, + }, + { + name: "non-existing file /src/ with mustExist=false", + mustExist: false, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + src = filepath.Join(tmpdir, "src") + return src, src, nil + }, + }, + { + name: "non-existing file /existing/path/src with mustExist=false with existing directories", + mustExist: false, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + src = filepath.Join(tmpdir, "existing/path") + if err := os.MkdirAll(src, 0755); err != nil { + return "", "", err + } + src = filepath.Join(src, "src") + return src, src, nil + }, + }, + { + name: "simple symlink /src -> /dst", + mustExist: false, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + dst := filepath.Join(tmpdir, "dst") + if err = ioutil.WriteFile(dst, []byte{}, 0644); err != nil { + return "", "", err + } + src = filepath.Join(tmpdir, "src") + err = os.Symlink(dst, src) + return src, dst, err + }, + }, + { + name: "dangling symlink /src -> /non-existing-path", + mustExist: true, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + dst := filepath.Join(tmpdir, "non-existing-path") + src = filepath.Join(tmpdir, "src") + err = os.Symlink(dst, src) + return src, "", err + }, + expectError: true, + }, + { + name: "dangling symlink /src -> /non-existing-path with mustExist=false", + mustExist: false, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + dst := filepath.Join(tmpdir, "non-existing-path") + src = filepath.Join(tmpdir, "src") + err = os.Symlink(dst, src) + return src, dst, err + }, + }, + { + name: "symlink to directory /src/file, where /src is link to /dst", + mustExist: true, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + dst := filepath.Join(tmpdir, "dst") + if err = os.Mkdir(dst, 0755); err != nil { + return "", "", err + } + dstFile := filepath.Join(dst, "file") + if err = ioutil.WriteFile(dstFile, []byte{}, 0644); err != nil { + return "", "", err + } + + src = filepath.Join(tmpdir, "src") + if err = os.Symlink(dst, src); err != nil { + return "", "", err + } + srcFile := filepath.Join(src, "file") + return srcFile, dstFile, nil + }, + }, + { + name: "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist", + mustExist: true, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + dst := filepath.Join(tmpdir, "dst") + + src = filepath.Join(tmpdir, "src") + if err = os.Symlink(dst, src); err != nil { + return "", "", err + } + srcFile := filepath.Join(src, "file") + return srcFile, "", nil + }, + expectError: true, + }, + { + name: "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist with mustExist=false", + mustExist: false, + prepare: func(tmpdir string) (src string, expectedDst string, err error) { + dst := filepath.Join(tmpdir, "dst") + dstFile := filepath.Join(dst, "file") + + src = filepath.Join(tmpdir, "src") + if err = os.Symlink(dst, src); err != nil { + return "", "", err + } + srcFile := filepath.Join(src, "file") + return srcFile, dstFile, nil + }, + }, + } + + for _, test := range tests { + ns := Nsenter{ + hostRootFsPath: "/rootfs", + executor: fakeExec{ + rootfsPath: "/rootfs", + }, + } + + tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + src, expectedDst, err := test.prepare(tmpdir) + if err != nil { + t.Error(err) + continue + } + + dst, err := ns.EvalSymlinks(src, test.mustExist) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("Test %q: expected error, got none", test.name) + } + if dst != expectedDst { + t.Errorf("Test %q: expected destination %q, got %q", test.name, expectedDst, dst) + } + } +} + +func TestNewNsenter(t *testing.T) { + // Create a symlink /tmp/xyz/rootfs -> / and use it as rootfs path + // It should resolve all binaries correctly, the test runs on Linux + + tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + rootfs := filepath.Join(tmpdir, "rootfs") + if err = os.Symlink("/", rootfs); err != nil { + t.Fatal(err) + } + + _, err = NewNsenter(rootfs, exec.New()) + if err != nil { + t.Errorf("Error: %s", err) + } +} + +func TestNewNsenterError(t *testing.T) { + // Create empty dir /tmp/xyz/rootfs and use it as rootfs path + // It should resolve all binaries correctly, the test runs on Linux + + tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + rootfs := filepath.Join(tmpdir, "rootfs") + if err = os.MkdirAll(rootfs, 0755); err != nil { + t.Fatal(err) + } + + _, err = NewNsenter(rootfs, exec.New()) + if err == nil { + t.Errorf("Expected error, got none") + } +} diff --git a/pkg/util/nsenter/nsenter_unsupported.go b/pkg/util/nsenter/nsenter_unsupported.go index 842cf046731..0618b9da469 100644 --- a/pkg/util/nsenter/nsenter_unsupported.go +++ b/pkg/util/nsenter/nsenter_unsupported.go @@ -22,6 +22,12 @@ import ( "k8s.io/utils/exec" ) +const ( + // DefaultHostRootFsPath is path to host's filesystem mounted into container + // with kubelet. + DefaultHostRootFsPath = "/rootfs" +) + // Nsenter is part of experimental support for running the kubelet // in a container. type Nsenter struct { @@ -30,7 +36,7 @@ type Nsenter struct { } // NewNsenter constructs a new instance of Nsenter -func NewNsenter() (*Nsenter, error) { +func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) { return &Nsenter{}, nil }