github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Log node info on test failures

Browse Source
This commit is contained in:
Brendan Burns
2015-11-04 14:07:49 -08:00
parent 17c5c0922b
commit b76fef7d48
3 changed files with 29 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/e2e/framework.go
View File

@@ -102,6 +102,8 @@ func (f *Framework) afterEach() {
// you may or may not see the killing/deletion/cleanup events.
dumpAllPodInfo(f.Client)
dumpAllNodeInfo(f.Client)
}
// Check whether all nodes are ready after the test.

19
test/e2e/reboot.go
View File

@@ -26,7 +26,6 @@ import (
"k8s.io/kubernetes/pkg/labels"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
@@ -44,13 +43,9 @@ const (
)
var _ = Describe("Reboot", func() {
var c *client.Client
f := NewFramework("reboot")
BeforeEach(func() {
var err error
c, err = loadClient()
Expect(err).NotTo(HaveOccurred())
// These tests requires SSH to nodes, so the provider check should be identical to there
// (the limiting factor is the implementation of util.go's getSigner(...)).
@@ -61,32 +56,32 @@ var _ = Describe("Reboot", func() {
It("each node by ordering clean reboot and ensure they function upon restart", func() {
// clean shutdown and restart
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is rebooted.
testReboot(c, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &")
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &")
})
It("each node by ordering unclean reboot and ensure they function upon restart", func() {
// unclean shutdown and restart
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown.
testReboot(c, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
testReboot(f.Client, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
})
It("each node by triggering kernel panic and ensure they function upon restart", func() {
// kernel panic
// We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered.
testReboot(c, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
testReboot(f.Client, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
})
It("each node by switching off the network interface and ensure they function upon switch on", func() {
// switch the network interface off for a while to simulate a network outage
// We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down.
testReboot(c, "nohup sh -c 'sleep 10 && sudo ifdown eth0 && sleep 120 && sudo ifup eth0' >/dev/null 2>&1 &")
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo ifdown eth0 && sleep 120 && sudo ifup eth0' >/dev/null 2>&1 &")
})
It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() {
// tell the firewall to drop all inbound packets for a while
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
// We still accept packages send from localhost to prevent monit from restarting kubelet.
testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+
" sleep 120 && sudo iptables -D INPUT -j DROP && sudo iptables -D INPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
})
@@ -94,7 +89,7 @@ var _ = Describe("Reboot", func() {
// tell the firewall to drop all outbound packets for a while
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets.
// We still accept packages send to localhost to prevent monit from restarting kubelet.
testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+
" sleep 120 && sudo iptables -D OUTPUT -j DROP && sudo iptables -D OUTPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
})
})

20
test/e2e/util.go
View File

@@ -1477,8 +1477,28 @@ func dumpAllPodInfo(c *client.Client) {
logPodStates(pods.Items)
}
func dumpAllNodeInfo(c *client.Client) {
nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
if err != nil {
Logf("unable to fetch node list: %v", err)
return
}
names := make([]string, len(nodes.Items))
for ix := range nodes.Items {
names[ix] = nodes.Items[ix].Name
}
dumpNodeDebugInfo(c, names)
}
func dumpNodeDebugInfo(c *client.Client, nodeNames []string) {
for _, n := range nodeNames {
Logf("\nLogging node info for node %v", n)
node, err := c.Nodes().Get(n)
if err != nil {
Logf("Error getting node info %v", err)
}
Logf("Node Info: %v", node)
Logf("\nLogging kubelet events for node %v", n)
for _, e := range getNodeEvents(c, n) {
Logf("source %v message %v reason %v first ts %v last ts %v, involved obj %+v",