Add QoS support on node

pkg/util/oom/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package oom implements utility functions relating to out of memory management.
+package oom

pkg/util/oom/oom.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oom
+
+// This is a struct instead of an interface to allow injection of process ID listers and
+// applying OOM score in tests.
+// TODO: make this an interface, and inject a mock ioutil struct for testing.
+type OomAdjuster struct {
+	pidLister                 func(cgroupName string) ([]int, error)
+	ApplyOomScoreAdj          func(pid int, oomScoreAdj int) error
+	ApplyOomScoreAdjContainer func(cgroupName string, oomScoreAdj, maxTries int) error
+}

pkg/util/oom/oom_fake.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oom
+
+type FakeOomAdjuster struct{}
+
+func NewFakeOomAdjuster() *OomAdjuster {
+	return &OomAdjuster{
+		ApplyOomScoreAdj:          fakeApplyOomScoreAdj,
+		ApplyOomScoreAdjContainer: fakeApplyOomScoreAdjContainer,
+	}
+}
+
+func fakeApplyOomScoreAdj(pid int, oomScoreAdj int) error {
+	return nil
+}
+
+func fakeApplyOomScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
+	return nil
+}

pkg/util/oom/oom_linux.go

@@ -0,0 +1,112 @@
+// +build cgo,linux
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oom
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strconv"
+
+	"github.com/docker/libcontainer/cgroups/fs"
+	"github.com/docker/libcontainer/configs"
+	"github.com/golang/glog"
+)
+
+func NewOomAdjuster() *OomAdjuster {
+	oomAdjuster := &OomAdjuster{
+		pidLister:        getPids,
+		ApplyOomScoreAdj: applyOomScoreAdj,
+	}
+	oomAdjuster.ApplyOomScoreAdjContainer = oomAdjuster.applyOomScoreAdjContainer
+	return oomAdjuster
+}
+
+func getPids(cgroupName string) ([]int, error) {
+	fsManager := fs.Manager{
+		Cgroups: &configs.Cgroup{
+			Name: cgroupName,
+		},
+	}
+	return fsManager.GetPids()
+}
+
+// Writes 'value' to /proc/<pid>/oom_score_adj. PID = 0 means self
+func applyOomScoreAdj(pid int, oomScoreAdj int) error {
+	if pid < 0 {
+		return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
+	}
+
+	var pidStr string
+	if pid == 0 {
+		pidStr = "self"
+	} else {
+		pidStr = strconv.Itoa(pid)
+	}
+
+	oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj")
+	maxTries := 2
+	var err error
+	for i := 0; i < maxTries; i++ {
+		_, readErr := ioutil.ReadFile(oomScoreAdjPath)
+		if readErr != nil {
+			err = fmt.Errorf("failed to read oom_score_adj: %v", readErr)
+		} else if writeErr := ioutil.WriteFile(oomScoreAdjPath, []byte(strconv.Itoa(oomScoreAdj)), 0700); writeErr != nil {
+			err = fmt.Errorf("failed to set oom_score_adj to %d: %v", oomScoreAdj, writeErr)
+		} else {
+			return nil
+		}
+	}
+
+	return err
+}
+
+// Writes 'value' to /proc/<pid>/oom_score_adj for all processes in cgroup cgroupName.
+// Keeps trying to write until the process list of the cgroup stabilizes, or until maxTries tries.
+func (oomAdjuster *OomAdjuster) applyOomScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
+	adjustedProcessSet := make(map[int]bool)
+	for i := 0; i < maxTries; i++ {
+		continueAdjusting := false
+		pidList, err := oomAdjuster.pidLister(cgroupName)
+		if err != nil {
+			continueAdjusting = true
+			glog.Errorf("Error getting process list for cgroup %s: %+v", cgroupName, err)
+		} else if len(pidList) == 0 {
+			continueAdjusting = true
+		} else {
+			for _, pid := range pidList {
+				if !adjustedProcessSet[pid] {
+					continueAdjusting = true
+					if err = oomAdjuster.ApplyOomScoreAdj(pid, oomScoreAdj); err == nil {
+						adjustedProcessSet[pid] = true
+					}
+				}
+			}
+		}
+		if !continueAdjusting {
+			return nil
+		}
+		// There's a slight race. A process might have forked just before we write its OOM score adjust.
+		// The fork might copy the parent process's old OOM score, then this function might execute and
+		// update the parent's OOM score, but the forked process id might not be reflected in cgroup.procs
+		// for a short amount of time. So this function might return without changing the forked process's
+		// OOM score. Very unlikely race, so ignoring this for now.
+	}
+	return fmt.Errorf("exceeded maxTries, some processes might not have desired OOM score")
+}

pkg/util/oom/oom_linux_test.go

@@ -0,0 +1,110 @@
+// +build cgo,linux
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oom
+
+import (
+	"testing"
+)
+
+// Converts a sequence of PID lists into a PID lister.
+// The PID lister returns pidListSequence[i] on the ith call. If i >= length of pidListSequence
+// then return the last element of pidListSequence (the sequence is considered to have) stabilized.
+func sequenceToPidLister(pidListSequence [][]int) func(string) ([]int, error) {
+	var numCalls int
+	return func(cgroupName string) ([]int, error) {
+		numCalls++
+		if len(pidListSequence) == 0 {
+			return []int{}, nil
+		} else if numCalls > len(pidListSequence) {
+			return pidListSequence[len(pidListSequence)-1], nil
+		}
+		return pidListSequence[numCalls-1], nil
+	}
+}
+
+// Tests that applyOomScoreAdjContainer correctly applies OOM scores to relevant processes, or
+// returns the right error.
+func applyOomScoreAdjContainerTester(pidListSequence [][]int, maxTries int, appliedPids []int, expectedError bool, t *testing.T) {
+	pidOoms := make(map[int]bool)
+
+	// Mock ApplyOomScoreAdj and pidLister.
+	oomAdjuster := NewOomAdjuster()
+	oomAdjuster.ApplyOomScoreAdj = func(pid int, oomScoreAdj int) error {
+		pidOoms[pid] = true
+		return nil
+	}
+	oomAdjuster.pidLister = sequenceToPidLister(pidListSequence)
+	err := oomAdjuster.ApplyOomScoreAdjContainer("", 100, maxTries)
+
+	// Check error value.
+	if expectedError && err == nil {
+		t.Errorf("Expected error %+v when running ApplyOomScoreAdjContainer but got no error", expectedError)
+		return
+	} else if !expectedError && err != nil {
+		t.Errorf("Expected no error but got error %+v when running ApplyOomScoreAdjContainer", err)
+		return
+	} else if err != nil {
+		return
+	}
+
+	// Check that OOM scores were applied to the right processes.
+	if len(appliedPids) != len(pidOoms) {
+		t.Errorf("Applied OOM scores to incorrect number of processes")
+		return
+	}
+	for _, pid := range appliedPids {
+		if !pidOoms[pid] {
+			t.Errorf("Failed to apply OOM scores to process %d", pid)
+		}
+	}
+}
+
+func TestOomScoreAdjContainer(t *testing.T) {
+	pidListSequenceEmpty := [][]int{}
+	applyOomScoreAdjContainerTester(pidListSequenceEmpty, 3, nil, true, t)
+
+	pidListSequence1 := [][]int{
+		{1, 2},
+	}
+	applyOomScoreAdjContainerTester(pidListSequence1, 1, nil, true, t)
+	applyOomScoreAdjContainerTester(pidListSequence1, 2, []int{1, 2}, false, t)
+	applyOomScoreAdjContainerTester(pidListSequence1, 3, []int{1, 2}, false, t)
+
+	pidListSequence3 := [][]int{
+		{1, 2},
+		{1, 2, 4, 5},
+		{2, 1, 4, 5, 3},
+	}
+	applyOomScoreAdjContainerTester(pidListSequence3, 1, nil, true, t)
+	applyOomScoreAdjContainerTester(pidListSequence3, 2, nil, true, t)
+	applyOomScoreAdjContainerTester(pidListSequence3, 3, nil, true, t)
+	applyOomScoreAdjContainerTester(pidListSequence3, 4, []int{1, 2, 3, 4, 5}, false, t)
+
+	pidListSequenceLag := [][]int{
+		{},
+		{},
+		{},
+		{1, 2, 4},
+		{1, 2, 4, 5},
+	}
+	for i := 1; i < 5; i++ {
+		applyOomScoreAdjContainerTester(pidListSequenceLag, i, nil, true, t)
+	}
+	applyOomScoreAdjContainerTester(pidListSequenceLag, 6, []int{1, 2, 4, 5}, false, t)
+}

pkg/util/oom/oom_unsupported.go

@@ -0,0 +1,40 @@
+// +build !cgo !linux
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oom
+
+import (
+	"errors"
+)
+
+var unsupportedErr = errors.New("setting OOM scores is unsupported in this build")
+
+func NewOomAdjuster() *OomAdjuster {
+	return &OomAdjuster{
+		ApplyOomScoreAdj:          unsupportedApplyOomScoreAdj,
+		ApplyOomScoreAdjContainer: unsupportedApplyOomScoreAdjContainer,
+	}
+}
+
+func unsupportedApplyOomScoreAdj(pid int, oomScoreAdj int) error {
+	return unsupportedErr
+}
+
+func unsupportedApplyOomScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
+	return unsupportedErr
+}

pkg/util/procfs/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package procfs implements utility functions relating to the /proc mount.
+package procfs

pkg/util/procfs/example_proc_cgroup

@@ -0,0 +1,10 @@
+11:name=systemd:/user/1000.user/c1.session
+10:hugetlb:/user/1000.user/c1.session
+9:perf_event:/user/1000.user/c1.session
+8:blkio:/user/1000.user/c1.session
+7:freezer:/user/1000.user/c1.session
+6:devices:/user/1000.user/c1.session
+5:memory:/user/1000.user/c1.session
+4:cpuacct:/user/1000.user/c1.session
+3:cpu:/user/1000.user/c1.session
+2:cpuset:/

pkg/util/procfs/procfs.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package procfs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strconv"
+	"strings"
+)
+
+type ProcFs struct{}
+
+func NewProcFs() ProcFsInterface {
+	return &ProcFs{}
+}
+
+func containerNameFromProcCgroup(content string) (string, error) {
+	lines := strings.Split(content, "\n")
+	for _, line := range lines {
+		entries := strings.SplitN(line, ":", 3)
+		if len(entries) == 3 && entries[1] == "devices" {
+			return strings.TrimSpace(entries[2]), nil
+		}
+	}
+	return "", fmt.Errorf("could not find devices cgroup location")
+}
+
+// getFullContainerName gets the container name given the root process id of the container.
+// Eg. If the devices cgroup for the container is stored in /sys/fs/cgroup/devices/docker/nginx,
+// return docker/nginx. Assumes that the process is part of exactly one cgroup hierarchy.
+func (pfs *ProcFs) GetFullContainerName(pid int) (string, error) {
+	filePath := path.Join("/proc", strconv.Itoa(pid), "cgroup")
+	content, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return "", err
+	}
+	return containerNameFromProcCgroup(string(content))
+}

pkg/util/procfs/procfs_fake.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package procfs
+
+type FakeProcFs struct{}
+
+func NewFakeProcFs() ProcFsInterface {
+	return &FakeProcFs{}
+}
+
+// getFullContainerName gets the container name given the root process id of the container.
+// Eg. If the devices cgroup for the container is stored in /sys/fs/cgroup/devices/docker/nginx,
+// return docker/nginx. Assumes that the process is part of exactly one cgroup hierarchy.
+func (fakePfs *FakeProcFs) GetFullContainerName(pid int) (string, error) {
+	return "", nil
+}

pkg/util/procfs/procfs_interface.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package procfs
+
+type ProcFsInterface interface {
+	// getFullContainerName gets the container name given the root process id of the container.
+	GetFullContainerName(pid int) (string, error)
+}

pkg/util/procfs/procfs_test.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package procfs
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+func verifyContainerName(procCgroupText, expectedName string, expectedErr bool, t *testing.T) {
+	name, err := containerNameFromProcCgroup(procCgroupText)
+	if expectedErr && err == nil {
+		t.Errorf("Expected error but did not get error in verifyContainerName")
+		return
+	} else if !expectedErr && err != nil {
+		t.Errorf("Expected no error, but got error %+v in verifyContainerName", err)
+		return
+	} else if expectedErr {
+		return
+	}
+	if name != expectedName {
+		t.Errorf("Expected container name %s but got name %s", expectedName, name)
+	}
+}
+
+func TestContainerNameFromProcCgroup(t *testing.T) {
+	procCgroupValid := "2:devices:docker/kubelet"
+	verifyContainerName(procCgroupValid, "docker/kubelet", false, t)
+
+	procCgroupEmpty := ""
+	verifyContainerName(procCgroupEmpty, "", true, t)
+
+	content, err := ioutil.ReadFile("example_proc_cgroup")
+	if err != nil {
+		t.Errorf("Could not read example /proc cgroup file")
+	}
+	verifyContainerName(string(content), "/user/1000.user/c1.session", false, t)
+
+	procCgroupNoDevice := "2:freezer:docker/kubelet\n5:cpuacct:pkg/kubectl"
+	verifyContainerName(procCgroupNoDevice, "", true, t)
+
+	procCgroupInvalid := "devices:docker/kubelet\ncpuacct:pkg/kubectl"
+	verifyContainerName(procCgroupInvalid, "", true, t)
+}

pkg/util/util.go

@@ -22,7 +22,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -212,34 +211,6 @@ func UsingSystemdInitSystem() bool {
 	return false
 }
 
-// Writes 'value' to /proc/<pid>/oom_score_adj. PID = 0 means self
-func ApplyOomScoreAdj(pid int, value int) error {
-	if value < -1000 || value > 1000 {
-		return fmt.Errorf("invalid value(%d) specified for oom_score_adj. Values must be within the range [-1000, 1000]", value)
-	}
-	if pid < 0 {
-		return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
-	}
-
-	var pidStr string
-	if pid == 0 {
-		pidStr = "self"
-	} else {
-		pidStr = strconv.Itoa(pid)
-	}
-
-	oom_value, err := ioutil.ReadFile(path.Join("/proc", pidStr, "oom_score_adj"))
-	if err != nil {
-		return fmt.Errorf("failed to read oom_score_adj: %v", err)
-	} else if string(oom_value) != strconv.Itoa(value) {
-		if err := ioutil.WriteFile(path.Join("/proc", pidStr, "oom_score_adj"), []byte(strconv.Itoa(value)), 0700); err != nil {
-			return fmt.Errorf("failed to set oom_score_adj to %d: %v", value, err)
-		}
-	}
-
-	return nil
-}
-
 // Tests whether all pointer fields in a struct are nil.  This is useful when,
 // for example, an API struct is handled by plugins which need to distinguish
 // "no plugin accepted this spec" from "this spec is empty".