github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Better kubelet logging for probes

Browse Source 
Log when we actually run probes and event when they fail.  Print the output of
a probe, too.
This commit is contained in:
Tim Hockin
2015-05-13 17:30:37 -07:00
parent d85dc7b2ea
commit 75617e8760
11 changed files with 162 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/probe/exec/exec.go
View File

@@ -28,24 +28,24 @@ func New() ExecProber {
}
type ExecProber interface {
Probe(e exec.Cmd) (probe.Result, error)
Probe(e exec.Cmd) (probe.Result, string, error)
}
type execProber struct{}
func (pr execProber) Probe(e exec.Cmd) (probe.Result, error) {
func (pr execProber) Probe(e exec.Cmd) (probe.Result, string, error) {
data, err := e.CombinedOutput()
glog.V(4).Infof("health check response: %s", string(data))
glog.V(4).Infof("Exec probe response: %q", string(data))
if err != nil {
exit, ok := err.(exec.ExitError)
if ok {
if exit.ExitStatus() == 0 {
return probe.Success, nil
return probe.Success, string(data), nil
} else {
return probe.Failure, nil
return probe.Failure, string(data), nil
}
}
return probe.Unknown, err
return probe.Unknown, "", err
}
return probe.Success, nil
return probe.Success, string(data), nil
}

42
pkg/probe/exec/exec_test.go
View File

@@ -34,13 +34,6 @@ func (f *FakeCmd) CombinedOutput() ([]byte, error) {
func (f *FakeCmd) SetDir(dir string) {}
type healthCheckTest struct {
expectedStatus probe.Result
expectError bool
output []byte
err error
}
type fakeExitError struct {
exited bool
statusCode int
@@ -64,30 +57,39 @@ func (f *fakeExitError) ExitStatus() int {
func TestExec(t *testing.T) {
prober := New()
fake := FakeCmd{}
tests := []healthCheckTest{
tests := []struct {
expectedStatus probe.Result
expectError bool
output string
err error
}{
// Ok
{probe.Success, false, []byte("OK"), nil},
{probe.Success, false, "OK", nil},
// Ok
{probe.Success, false, []byte("OK"), &fakeExitError{true, 0}},
{probe.Success, false, "OK", &fakeExitError{true, 0}},
// Run returns error
{probe.Unknown, true, []byte("OK, NOT"), fmt.Errorf("test error")},
{probe.Unknown, true, "", fmt.Errorf("test error")},
// Unhealthy
{probe.Failure, false, []byte("Fail"), &fakeExitError{true, 1}},
{probe.Failure, false, "Fail", &fakeExitError{true, 1}},
}
for _, test := range tests {
fake.out = test.output
fake.err = test.err
status, err := prober.Probe(&fake)
for i, test := range tests {
fake := FakeCmd{
out: []byte(test.output),
err: test.err,
}
status, output, err := prober.Probe(&fake)
if status != test.expectedStatus {
t.Errorf("expected %v, got %v", test.expectedStatus, status)
t.Errorf("[%d] expected %v, got %v", i, test.expectedStatus, status)
}
if err != nil && test.expectError == false {
t.Errorf("unexpected error: %v", err)
t.Errorf("[%d] unexpected error: %v", i, err)
}
if err == nil && test.expectError == true {
t.Errorf("unexpected non-error")
t.Errorf("[%d] unexpected non-error", i)
}
if test.output != output {
t.Errorf("[%d] expected %s, got %s", i, test.output, output)
}
}
}

24
pkg/probe/http/http.go
View File

@@ -17,6 +17,7 @@ limitations under the License.
package http
import (
"io/ioutil"
"net"
"net/http"
"net/url"
@@ -34,7 +35,7 @@ func New() HTTPProber {
}
type HTTPProber interface {
Probe(host string, port int, path string, timeout time.Duration) (probe.Result, error)
Probe(host string, port int, path string, timeout time.Duration) (probe.Result, string, error)
}
type httpProber struct {
@@ -42,7 +43,7 @@ type httpProber struct {
}
// Probe returns a ProbeRunner capable of running an http check.
func (pr httpProber) Probe(host string, port int, path string, timeout time.Duration) (probe.Result, error) {
func (pr httpProber) Probe(host string, port int, path string, timeout time.Duration) (probe.Result, string, error) {
return DoHTTPProbe(formatURL(host, port, path), &http.Client{Timeout: timeout, Transport: pr.transport})
}
@@ -54,18 +55,23 @@ type HTTPGetInterface interface {
// If the HTTP response code is successful (i.e. 400 > code >= 200), it returns Success.
// If the HTTP response code is unsuccessful or HTTP communication fails, it returns Failure.
// This is exported because some other packages may want to do direct HTTP probes.
func DoHTTPProbe(url string, client HTTPGetInterface) (probe.Result, error) {
func DoHTTPProbe(url string, client HTTPGetInterface) (probe.Result, string, error) {
res, err := client.Get(url)
if err != nil {
glog.V(1).Infof("HTTP probe error: %v", err)
return probe.Failure, nil
// Convert errors into failures to catch timeouts.
return probe.Failure, err.Error(), nil
}
defer res.Body.Close()
if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
return probe.Success, nil
b, err := ioutil.ReadAll(res.Body)
if err != nil {
return probe.Failure, "", err
}
glog.V(1).Infof("Health check failed for %s, Response: %v", url, *res)
return probe.Failure, nil
body := string(b)
if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
return probe.Success, body, nil
}
glog.V(4).Infof("Probe failed for %s, Response: %v", url, *res)
return probe.Failure, body, nil
}
// formatURL formats a URL from args. For testability.

20
pkg/probe/http/http_test.go
View File

@@ -22,6 +22,7 @@ import (
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
@@ -47,19 +48,23 @@ func TestFormatURL(t *testing.T) {
}
func TestHTTPProbeChecker(t *testing.T) {
handleReq := func(s int) func(w http.ResponseWriter) {
return func(w http.ResponseWriter) { w.WriteHeader(s) }
handleReq := func(s int, body string) func(w http.ResponseWriter) {
return func(w http.ResponseWriter) {
w.WriteHeader(s)
w.Write([]byte(body))
}
}
prober := New()
testCases := []struct {
handler func(w http.ResponseWriter)
health probe.Result
body string
}{
// The probe will be filled in below. This is primarily testing that an HTTP GET happens.
{handleReq(http.StatusOK), probe.Success},
{handleReq(-1), probe.Failure},
{func(w http.ResponseWriter) { time.Sleep(3 * time.Second) }, probe.Failure},
{handleReq(http.StatusOK, "ok body"), probe.Success, "ok body"},
{handleReq(-1, "fail body"), probe.Failure, "fail body"},
{func(w http.ResponseWriter) { time.Sleep(3 * time.Second) }, probe.Failure, "use of closed network connection"},
}
for _, test := range testCases {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -77,7 +82,7 @@ func TestHTTPProbeChecker(t *testing.T) {
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
health, err := prober.Probe(host, p, "", 1*time.Second)
health, output, err := prober.Probe(host, p, "", 1*time.Second)
if test.health == probe.Unknown && err == nil {
t.Errorf("Expected error")
}
@@ -87,5 +92,8 @@ func TestHTTPProbeChecker(t *testing.T) {
if health != test.health {
t.Errorf("Expected %v, got %v", test.health, health)
}
if !strings.Contains(output, test.body) {
t.Errorf("Expected %v, got %v", test.body, output)
}
}
}

13
pkg/probe/tcp/tcp.go
View File

@@ -31,12 +31,12 @@ func New() TCPProber {
}
type TCPProber interface {
Probe(host string, port int, timeout time.Duration) (probe.Result, error)
Probe(host string, port int, timeout time.Duration) (probe.Result, string, error)
}
type tcpProber struct{}
func (pr tcpProber) Probe(host string, port int, timeout time.Duration) (probe.Result, error) {
func (pr tcpProber) Probe(host string, port int, timeout time.Duration) (probe.Result, string, error) {
return DoTCPProbe(net.JoinHostPort(host, strconv.Itoa(port)), timeout)
}
@@ -44,14 +44,15 @@ func (pr tcpProber) Probe(host string, port int, timeout time.Duration) (probe.R
// If the socket can be opened, it returns Success
// If the socket fails to open, it returns Failure.
// This is exported because some other packages may want to do direct TCP probes.
func DoTCPProbe(addr string, timeout time.Duration) (probe.Result, error) {
func DoTCPProbe(addr string, timeout time.Duration) (probe.Result, string, error) {
conn, err := net.DialTimeout("tcp", addr, timeout)
if err != nil {
return probe.Failure, nil
// Convert errors to failures to handle timeouts.
return probe.Failure, err.Error(), nil
}
err = conn.Close()
if err != nil {
glog.Errorf("unexpected error closing health check socket: %v (%#v)", err, err)
glog.Errorf("Unexpected error closing TCP probe socket: %v (%#v)", err, err)
}
return probe.Success, nil
return probe.Success, "", nil
}

11
pkg/probe/tcp/tcp_test.go
View File

@@ -22,6 +22,7 @@ import (
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
@@ -34,10 +35,11 @@ func TestTcpHealthChecker(t *testing.T) {
expectedStatus probe.Result
usePort bool
expectError bool
output string
}{
// The probe will be filled in below. This is primarily testing that a connection is made.
{probe.Success, true, false},
{probe.Failure, false, false},
{probe.Success, true, false, ""},
{probe.Failure, false, false, "tcp: unknown port"},
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -60,7 +62,7 @@ func TestTcpHealthChecker(t *testing.T) {
if !test.usePort {
p = -1
}
status, err := prober.Probe(host, p, 1*time.Second)
status, output, err := prober.Probe(host, p, 1*time.Second)
if status != test.expectedStatus {
t.Errorf("expected: %v, got: %v", test.expectedStatus, status)
}
@@ -70,5 +72,8 @@ func TestTcpHealthChecker(t *testing.T) {
if err == nil && test.expectError {
t.Errorf("unexpected non-error.")
}
if !strings.Contains(output, test.output) {
t.Errorf("expected %s, got %s", test.output, output)
}
}
}