0b8244505c
This test tracks kubelet resource usage over a long period of time (1hr) when running N pods (e.g., N=0,50), and prints out the resource usage. This would give us an idea how much kubelet's management overhead is in a stable cluster. Some followup items: * Use a more realistic workload (e.g., including probing) * Fail the test if the resource usage is too high. Caveat: * We assume the scheduler would do a decent job distributing the pause pods, but we should double check. * Cluster addon pods could be unevenly distributed and skews the resource usage on nodes.
122 lines
3.6 KiB
Go
122 lines
3.6 KiB
Go
/*
|
|
Copyright 2015 The Kubernetes Authors All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package e2e
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
client "k8s.io/kubernetes/pkg/client/unversioned"
|
|
"k8s.io/kubernetes/pkg/fields"
|
|
"k8s.io/kubernetes/pkg/labels"
|
|
"k8s.io/kubernetes/pkg/util"
|
|
"k8s.io/kubernetes/pkg/util/sets"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
const (
|
|
// Interval to poll /stats/container on a node
|
|
containerStatsPollingPeriod = 10 * time.Second
|
|
// The monitoring time for one test.
|
|
monitoringTime = 30 * time.Minute
|
|
// The periodic reporting period.
|
|
reportingPeriod = 5 * time.Minute
|
|
)
|
|
|
|
func logPodsOnNodes(c *client.Client, nodeNames []string) {
|
|
for _, n := range nodeNames {
|
|
podList, err := GetKubeletPods(c, n)
|
|
if err != nil {
|
|
Logf("Unable to retrieve kubelet pods for node %v", n)
|
|
continue
|
|
}
|
|
Logf("%d pods are running on node %v", len(podList.Items), n)
|
|
}
|
|
}
|
|
|
|
var _ = Describe("Kubelet", func() {
|
|
var numNodes int
|
|
var nodeNames sets.String
|
|
framework := NewFramework("kubelet-perf")
|
|
var resourceMonitor *resourceMonitor
|
|
|
|
BeforeEach(func() {
|
|
nodes, err := framework.Client.Nodes().List(labels.Everything(), fields.Everything())
|
|
expectNoError(err)
|
|
numNodes = len(nodes.Items)
|
|
nodeNames = sets.NewString()
|
|
for _, node := range nodes.Items {
|
|
nodeNames.Insert(node.Name)
|
|
}
|
|
resourceMonitor = newResourceMonitor(framework.Client, targetContainers(), containerStatsPollingPeriod)
|
|
resourceMonitor.Start()
|
|
})
|
|
|
|
AfterEach(func() {
|
|
resourceMonitor.Stop()
|
|
})
|
|
|
|
Describe("resource usage tracking", func() {
|
|
density := []int{0, 50}
|
|
for _, podsPerNode := range density {
|
|
name := fmt.Sprintf(
|
|
"over %v with %d pods per node.", monitoringTime, podsPerNode)
|
|
It(name, func() {
|
|
totalPods := podsPerNode * numNodes
|
|
By(fmt.Sprintf("Creating a RC of %d pods and wait until all pods of this RC are running", totalPods))
|
|
rcName := fmt.Sprintf("resource%d-%s", totalPods, string(util.NewUUID()))
|
|
|
|
// TODO: Use a more realistic workload
|
|
Expect(RunRC(RCConfig{
|
|
Client: framework.Client,
|
|
Name: rcName,
|
|
Namespace: framework.Namespace.Name,
|
|
Image: "gcr.io/google_containers/pause:go",
|
|
Replicas: totalPods,
|
|
})).NotTo(HaveOccurred())
|
|
|
|
// Log once and flush the stats.
|
|
resourceMonitor.LogLatest()
|
|
resourceMonitor.Reset()
|
|
|
|
By("Start monitoring resource usage")
|
|
// Periodically dump the cpu summary until the deadline is met.
|
|
// Note that without calling resourceMonitor.Reset(), the stats
|
|
// would occupy increasingly more memory. This should be fine
|
|
// for the current test duration, but we should reclaim the
|
|
// entries if we plan to monitor longer (e.g., 8 hours).
|
|
deadline := time.Now().Add(monitoringTime)
|
|
for time.Now().Before(deadline) {
|
|
time.Sleep(reportingPeriod)
|
|
Logf("Still running...")
|
|
logPodsOnNodes(framework.Client, nodeNames.List())
|
|
}
|
|
|
|
By("Reporting overall resource usage")
|
|
logPodsOnNodes(framework.Client, nodeNames.List())
|
|
resourceMonitor.LogCPUSummary()
|
|
resourceMonitor.LogLatest()
|
|
|
|
By("Deleting the RC")
|
|
DeleteRC(framework.Client, framework.Namespace.Name, rcName)
|
|
})
|
|
}
|
|
})
|
|
})
|