diff --git a/pkg/volume/metrics_du.go b/pkg/volume/metrics_du.go index 1491c30de76..f080fac2b33 100644 --- a/pkg/volume/metrics_du.go +++ b/pkg/volume/metrics_du.go @@ -46,12 +46,7 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) { return metrics, NewNoPathDefinedError() } - err := md.runDiskUsage(metrics) - if err != nil { - return metrics, err - } - - err = md.runFind(metrics) + err := md.getDiskUsage(metrics) if err != nil { return metrics, err } @@ -64,23 +59,14 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) { return metrics, nil } -// runDiskUsage gets disk usage of md.path and writes the results to metrics.Used -func (md *metricsDu) runDiskUsage(metrics *Metrics) error { - used, err := fs.DiskUsage(md.path) +// getDiskUsage writes metrics.Used and metric.InodesUsed from fs.DiskUsage +func (md *metricsDu) getDiskUsage(metrics *Metrics) error { + usage, err := fs.DiskUsage(md.path) if err != nil { return err } - metrics.Used = used - return nil -} - -// runFind executes the "find" command and writes the results to metrics.InodesUsed -func (md *metricsDu) runFind(metrics *Metrics) error { - inodesUsed, err := fs.Find(md.path) - if err != nil { - return err - } - metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI) + metrics.Used = resource.NewQuantity(usage.Bytes, resource.BinarySI) + metrics.InodesUsed = resource.NewQuantity(usage.Inodes, resource.BinarySI) return nil } diff --git a/pkg/volume/util/fs/fs.go b/pkg/volume/util/fs/fs.go index a6114f259a6..77adf374785 100644 --- a/pkg/volume/util/fs/fs.go +++ b/pkg/volume/util/fs/fs.go @@ -19,17 +19,21 @@ limitations under the License. package fs import ( - "bytes" "fmt" - "os/exec" - "strings" + "os" + "path/filepath" + "syscall" "golang.org/x/sys/unix" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/volume/util/fsquota" ) +type UsageInfo struct { + Bytes int64 + Inodes int64 +} + // Info linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error) // for the filesystem that path resides upon. func Info(path string) (int64, int64, int64, int64, int64, int64, error) { @@ -55,63 +59,83 @@ func Info(path string) (int64, int64, int64, int64, int64, int64, error) { return available, capacity, usage, inodes, inodesFree, inodesUsed, nil } -// DiskUsage gets disk usage of specified path. -func DiskUsage(path string) (*resource.Quantity, error) { - // First check whether the quota system knows about this directory - // A nil quantity with no error means that the path does not support quotas - // and we should use other mechanisms. - data, err := fsquota.GetConsumption(path) - if data != nil { - return data, nil - } else if err != nil { - return nil, fmt.Errorf("unable to retrieve disk consumption via quota for %s: %v", path, err) - } - // Uses the same niceness level as cadvisor.fs does when running du - // Uses -B 1 to always scale to a blocksize of 1 byte - out, err := exec.Command("nice", "-n", "19", "du", "-x", "-s", "-B", "1", path).CombinedOutput() - if err != nil { - return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -x -s -B 1) on path %s with error %v", path, err) - } - used, err := resource.ParseQuantity(strings.Fields(string(out))[0]) - if err != nil { - return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err) - } - used.Format = resource.BinarySI - return &used, nil -} +// DiskUsage calculates the number of inodes and disk usage for a given directory +func DiskUsage(path string) (UsageInfo, error) { + var usage UsageInfo -// Find uses the equivalent of the command `find -dev -printf '.' | wc -c` to count files and directories. -// While this is not an exact measure of inodes used, it is a very good approximation. -func Find(path string) (int64, error) { if path == "" { - return 0, fmt.Errorf("invalid directory") + return usage, fmt.Errorf("invalid directory") } + // First check whether the quota system knows about this directory - // A nil quantity with no error means that the path does not support quotas - // and we should use other mechanisms. - inodes, err := fsquota.GetInodes(path) + // A nil quantity or error means that the path does not support quotas + // or xfs_quota tool is missing and we should use other mechanisms. + consumption, _ := fsquota.GetConsumption(path) + if consumption != nil { + usage.Bytes = consumption.Value() + } + + inodes, _ := fsquota.GetInodes(path) if inodes != nil { - return inodes.Value(), nil - } else if err != nil { - return 0, fmt.Errorf("unable to retrieve inode consumption via quota for %s: %v", path, err) + usage.Inodes = inodes.Value() } - var counter byteCounter - var stderr bytes.Buffer - findCmd := exec.Command("find", path, "-xdev", "-printf", ".") - findCmd.Stdout, findCmd.Stderr = &counter, &stderr - if err := findCmd.Start(); err != nil { - return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stderr.String()) - } - if err := findCmd.Wait(); err != nil { - return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stderr.String(), err) - } - return counter.bytesWritten, nil -} -// Simple io.Writer implementation that counts how many bytes were written. -type byteCounter struct{ bytesWritten int64 } + if inodes != nil && consumption != nil { + return usage, nil + } -func (b *byteCounter) Write(p []byte) (int, error) { - b.bytesWritten += int64(len(p)) - return len(p), nil + topLevelStat := &unix.Stat_t{} + err := unix.Stat(path, topLevelStat) + if err != nil { + return usage, err + } + + // dedupedInode stores inodes that could be duplicates (nlink > 1) + dedupedInodes := make(map[uint64]struct{}) + + err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + // ignore files that have been deleted after directory was read + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("unable to count inodes for %s: %s", path, err) + } + + // according to the docs, Sys can be nil + if info.Sys() == nil { + return fmt.Errorf("fileinfo Sys is nil") + } + + s, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("unsupported fileinfo; could not convert to stat_t") + } + + if s.Dev != topLevelStat.Dev { + // don't descend into directories on other devices + return filepath.SkipDir + } + + // Dedupe hardlinks + if s.Nlink > 1 { + if _, ok := dedupedInodes[s.Ino]; !ok { + dedupedInodes[s.Ino] = struct{}{} + } else { + return nil + } + } + + if consumption == nil { + usage.Bytes += int64(s.Blocks) * int64(512) // blocksize in bytes + } + + if inodes == nil { + usage.Inodes++ + } + + return nil + }) + + return usage, err } diff --git a/pkg/volume/util/fs/fs_unsupported.go b/pkg/volume/util/fs/fs_unsupported.go index 8cadf72bd53..6a098cb3f55 100644 --- a/pkg/volume/util/fs/fs_unsupported.go +++ b/pkg/volume/util/fs/fs_unsupported.go @@ -20,21 +20,20 @@ package fs import ( "fmt" - - "k8s.io/apimachinery/pkg/api/resource" ) +type UsageInfo struct { + Bytes int64 + Inodes int64 +} + // Info unsupported returns 0 values for available and capacity and an error. func Info(path string) (int64, int64, int64, int64, int64, int64, error) { return 0, 0, 0, 0, 0, 0, fmt.Errorf("fsinfo not supported for this build") } // DiskUsage gets disk usage of specified path. -func DiskUsage(path string) (*resource.Quantity, error) { - return nil, fmt.Errorf("du not supported for this build") -} - -// Find will always return zero since is on unsupported platform. -func Find(path string) (int64, error) { - return 0, fmt.Errorf("find not supported for this build") +func DiskUsage(path string) (UsageInfo, error) { + var usage UsageInfo + return usage, fmt.Errorf("directory disk usage not supported for this build.") } diff --git a/pkg/volume/util/fs/fs_windows.go b/pkg/volume/util/fs/fs_windows.go index 8d16eabcefe..356b2e223bf 100644 --- a/pkg/volume/util/fs/fs_windows.go +++ b/pkg/volume/util/fs/fs_windows.go @@ -26,8 +26,6 @@ import ( "unsafe" "golang.org/x/sys/windows" - - "k8s.io/apimachinery/pkg/api/resource" ) var ( @@ -35,6 +33,11 @@ var ( procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW") ) +type UsageInfo struct { + Bytes int64 + Inodes int64 +} + // Info returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error) // for the filesystem that path resides upon. func Info(path string) (int64, int64, int64, int64, int64, int64, error) { @@ -64,28 +67,15 @@ func Info(path string) (int64, int64, int64, int64, int64, int64, error) { } // DiskUsage gets disk usage of specified path. -func DiskUsage(path string) (*resource.Quantity, error) { +func DiskUsage(path string) (UsageInfo, error) { + var usage UsageInfo info, err := os.Lstat(path) if err != nil { - return nil, err + return usage, err } - usage, err := diskUsage(path, info) - if err != nil { - return nil, err - } - - used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage)) - if err != nil { - return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err) - } - used.Format = resource.BinarySI - return &used, nil -} - -// Find will always return zero since inodes is not supported on Windows. -func Find(path string) (int64, error) { - return 0, nil + usage.Bytes, err = diskUsage(path, info) + return usage, err } func diskUsage(currPath string, info os.FileInfo) (int64, error) { diff --git a/pkg/volume/util/fs/fs_windows_test.go b/pkg/volume/util/fs/fs_windows_test.go index df57d82b548..c9f69ba6941 100644 --- a/pkg/volume/util/fs/fs_windows_test.go +++ b/pkg/volume/util/fs/fs_windows_test.go @@ -67,10 +67,15 @@ func TestDiskUsage(t *testing.T) { } total := dirInfo1.Size() + dirInfo2.Size() + fileInfo1.Size() + fileInfo2.Size() - size, err := DiskUsage(dir1) + usage, err := DiskUsage(dir1) if err != nil { t.Fatalf("TestDiskUsage failed: %s", err.Error()) } + size, err := resource.ParseQuantity(fmt.Sprintf("%d", usage.Bytes)) + if err != nil { + t.Fatalf("TestDiskUsage failed: %s", err.Error()) + } + used, err := resource.ParseQuantity(fmt.Sprintf("%d", total)) if err != nil { t.Fatalf("TestDiskUsage failed: %s", err.Error())