Expose errors from libecycle hooks as events for the pod.

Signed-off-by: Vishnu kannan <vishnuk@google.com>

pkg/kubelet/container/event.go

@@ -62,4 +62,8 @@ const (
 
 	// Config event reason list
 	FailedValidation = "FailedValidation"
+
+	// Lifecycle hooks
+	FailedPostStartHook = "FailedPostStartHook"
+	FailedPreStopHook   = "FailedPreStopHook"
 )

pkg/kubelet/container/helpers.go

@@ -34,7 +34,7 @@ import (
 
 // HandlerRunner runs a lifecycle handler for a container.
 type HandlerRunner interface {
-	Run(containerID ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error
+	Run(containerID ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error)
 }
 
 // RuntimeHelper wraps kubelet to make container runtime

pkg/kubelet/dockertools/manager.go

@@ -1315,8 +1315,9 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co
 		go func() {
 			defer close(done)
 			defer utilruntime.HandleCrash()
-			if err := dm.runner.Run(containerID, pod, container, container.Lifecycle.PreStop); err != nil {
+			if msg, err := dm.runner.Run(containerID, pod, container, container.Lifecycle.PreStop); err != nil {
 				glog.Errorf("preStop hook for container %q failed: %v", name, err)
+				dm.generateFailedContainerEvent(containerID, pod.Name, kubecontainer.FailedPreStopHook, msg)
 			}
 		}()
 		select {
@@ -1362,6 +1363,15 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co
 	return err
 }
 
+func (dm *DockerManager) generateFailedContainerEvent(containerID kubecontainer.ContainerID, podName, reason, message string) {
+	ref, ok := dm.containerRefManager.GetRef(containerID)
+	if !ok {
+		glog.Warningf("No ref for pod '%q'", podName)
+		return
+	}
+	dm.recorder.Event(ref, api.EventTypeWarning, reason, message)
+}
+
 var errNoPodOnContainer = fmt.Errorf("no pod information labels on Docker container")
 
 // containerAndPodFromLabels tries to load the appropriate container info off of a Docker container's labels
@@ -1477,9 +1487,10 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe
 	}
 
 	if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {
-		handlerErr := dm.runner.Run(id, pod, container, container.Lifecycle.PostStart)
+		msg, handlerErr := dm.runner.Run(id, pod, container, container.Lifecycle.PostStart)
 		if handlerErr != nil {
 			err := fmt.Errorf("PostStart handler: %v", handlerErr)
+			dm.generateFailedContainerEvent(id, pod.Name, kubecontainer.FailedPostStartHook, msg)
 			dm.KillContainerInPod(id, container, pod, err.Error(), nil)
 			return kubecontainer.ContainerID{}, err
 		}

pkg/kubelet/lifecycle/fake_handler_runner.go

@@ -35,12 +35,12 @@ func NewFakeHandlerRunner() *FakeHandlerRunner {
 	return &FakeHandlerRunner{HandlerRuns: []string{}}
 }
 
-func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error {
+func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) {
 	hr.Lock()
 	defer hr.Unlock()
 
 	if hr.Err != nil {
-		return hr.Err
+		return "", hr.Err
 	}
 
 	switch {
@@ -51,9 +51,9 @@ func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api
 	case handler.TCPSocket != nil:
 		hr.HandlerRuns = append(hr.HandlerRuns, fmt.Sprintf("tcp-socket on pod: %v, container: %v: %v", format.Pod(pod), container.Name, containerID.String()))
 	default:
-		return fmt.Errorf("Invalid handler: %v", handler)
+		return "", fmt.Errorf("Invalid handler: %v", handler)
 	}
-	return nil
+	return "", nil
 }
 
 func (hr *FakeHandlerRunner) Reset() {

pkg/kubelet/lifecycle/handlers.go

@@ -52,26 +52,32 @@ func NewHandlerRunner(httpGetter kubetypes.HttpGetter, commandRunner kubecontain
 	}
 }
 
-func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error {
+func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) {
 	switch {
 	case handler.Exec != nil:
-		var buffer bytes.Buffer
+		var (
+			buffer bytes.Buffer
+			msg    string
+		)
 		output := ioutils.WriteCloserWrapper(&buffer)
 		err := hr.commandRunner.ExecInContainer(containerID, handler.Exec.Command, nil, output, output, false)
 		if err != nil {
-			glog.V(1).Infof("Exec lifecycle hook (%v) for Container %q in Pod %q failed - %q", handler.Exec.Command, container.Name, format.Pod(pod), buffer.String())
+			msg := fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - %q", handler.Exec.Command, container.Name, format.Pod(pod), buffer.String())
+			glog.V(1).Infof(msg)
 		}
-		return err
+		return msg, err
 	case handler.HTTPGet != nil:
 		msg, err := hr.runHTTPHandler(pod, container, handler)
 		if err != nil {
-			glog.V(1).Infof("Http lifecycle hook (%s) for Container %q in Pod %q failed - %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), msg)
+			msg := fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), msg)
+			glog.V(1).Infof(msg)
 		}
-		return err
+		return msg, err
 	default:
 		err := fmt.Errorf("Invalid handler: %v", handler)
-		glog.Errorf("Cannot run handler: %v", err)
+		msg := fmt.Sprintf("Cannot run handler: %v", err)
-		return err
+		glog.Errorf(msg)
+		return msg, err
 	}
 }
 

pkg/kubelet/lifecycle/handlers_test.go

@@ -19,8 +19,10 @@ package lifecycle
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"reflect"
+	"strings"
 	"testing"
 
 	"k8s.io/kubernetes/pkg/api"
@@ -110,7 +112,7 @@ func TestRunHandlerExec(t *testing.T) {
 	pod.ObjectMeta.Name = "podFoo"
 	pod.ObjectMeta.Namespace = "nsFoo"
 	pod.Spec.Containers = []api.Container{container}
-	err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
+	_, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -123,11 +125,12 @@ func TestRunHandlerExec(t *testing.T) {
 type fakeHTTP struct {
 	url  string
 	err  error
+	resp *http.Response
 }
 
 func (f *fakeHTTP) Get(url string) (*http.Response, error) {
 	f.url = url
-	return nil, f.err
+	return f.resp, f.err
 }
 
 func TestRunHandlerHttp(t *testing.T) {
@@ -153,7 +156,7 @@ func TestRunHandlerHttp(t *testing.T) {
 	pod.ObjectMeta.Name = "podFoo"
 	pod.ObjectMeta.Namespace = "nsFoo"
 	pod.Spec.Containers = []api.Container{container}
-	err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
+	_, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
 
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
@@ -180,7 +183,7 @@ func TestRunHandlerNil(t *testing.T) {
 	pod.ObjectMeta.Name = podName
 	pod.ObjectMeta.Namespace = podNamespace
 	pod.Spec.Containers = []api.Container{container}
-	err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
+	_, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
 	if err == nil {
 		t.Errorf("expect error, but got nil")
 	}
@@ -188,14 +191,13 @@ func TestRunHandlerNil(t *testing.T) {
 
 func TestRunHandlerHttpFailure(t *testing.T) {
 	expectedErr := fmt.Errorf("fake http error")
-	fakeHttp := fakeHTTP{err: expectedErr}
+	expectedResp := http.Response{
-	handlerRunner := &HandlerRunner{
+		Body: ioutil.NopCloser(strings.NewReader(expectedErr.Error())),
-		httpGetter:       &fakeHttp,
-		commandRunner:    &fakeContainerCommandRunner{},
-		containerManager: nil,
 	}
+	fakeHttp := fakeHTTP{err: expectedErr, resp: &expectedResp}
+	handlerRunner := NewHandlerRunner(&fakeHttp, &fakeContainerCommandRunner{}, nil)
 	containerName := "containerFoo"
-
+	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
 	container := api.Container{
 		Name: containerName,
 		Lifecycle: &api.Lifecycle{
@@ -212,12 +214,12 @@ func TestRunHandlerHttpFailure(t *testing.T) {
 	pod.ObjectMeta.Name = "podFoo"
 	pod.ObjectMeta.Namespace = "nsFoo"
 	pod.Spec.Containers = []api.Container{container}
-	msg, err := handlerRunner.runHTTPHandler(&pod, &container, container.Lifecycle.PostStart)
+	msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
 	if err == nil {
 		t.Errorf("expected error: %v", expectedErr)
 	}
-	if msg != "" {
+	if msg != expectedErr.Error() {
-		t.Errorf("expected empty error message")
+		t.Errorf("unexpected error message: %q; expected %q", msg, expectedErr)
 	}
 	if fakeHttp.url != "http://foo:8080/bar" {
 		t.Errorf("unexpected url: %s", fakeHttp.url)

pkg/kubelet/rkt/rkt.go

@@ -1184,7 +1184,16 @@ func (r *Runtime) RunPod(pod *api.Pod, pullSecrets []api.Secret) error {
 
 func (r *Runtime) runPreStopHook(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container) error {
 	glog.V(4).Infof("rkt: Running pre-stop hook for container %q of pod %q", container.Name, format.Pod(pod))
-	return r.runner.Run(containerID, pod, container, container.Lifecycle.PreStop)
+	msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PreStop)
+	if err != nil {
+		ref, ok := r.containerRefManager.GetRef(containerID)
+		if !ok {
+			glog.Warningf("No ref for container %q", containerID)
+		} else {
+			r.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedPreStopHook, msg)
+		}
+	}
+	return err
 }
 
 func (r *Runtime) runPostStartHook(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container) error {
@@ -1215,7 +1224,16 @@ func (r *Runtime) runPostStartHook(containerID kubecontainer.ContainerID, pod *a
 		return fmt.Errorf("rkt: Pod %q doesn't become running in %v: %v", format.Pod(pod), timeout, err)
 	}
 
-	return r.runner.Run(containerID, pod, container, container.Lifecycle.PostStart)
+	msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PostStart)
+	if err != nil {
+		ref, ok := r.containerRefManager.GetRef(containerID)
+		if !ok {
+			glog.Warningf("No ref for container %q", containerID)
+		} else {
+			r.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedPostStartHook, msg)
+		}
+	}
+	return err
 }
 
 type lifecycleHookType string