github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Expose errors from libecycle hooks as events for the pod.

Browse Source 
Signed-off-by: Vishnu kannan <vishnuk@google.com>
This commit is contained in:
Vishnu kannan
2016-05-23 13:19:54 -07:00
parent 3905179fd7
commit e7721ea8cf
7 changed files with 73 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/kubelet/container/event.go
View File

@@ -62,4 +62,8 @@ const (
// Config event reason list
FailedValidation = "FailedValidation"
// Lifecycle hooks
FailedPostStartHook = "FailedPostStartHook"
FailedPreStopHook = "FailedPreStopHook"
)

2
pkg/kubelet/container/helpers.go
View File

@@ -34,7 +34,7 @@ import (
// HandlerRunner runs a lifecycle handler for a container.
type HandlerRunner interface {
Run(containerID ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error
Run(containerID ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error)
}
// RuntimeHelper wraps kubelet to make container runtime

15
pkg/kubelet/dockertools/manager.go
View File

@@ -1315,8 +1315,9 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co
go func() {
defer close(done)
defer utilruntime.HandleCrash()
if err := dm.runner.Run(containerID, pod, container, container.Lifecycle.PreStop); err != nil {
if msg, err := dm.runner.Run(containerID, pod, container, container.Lifecycle.PreStop); err != nil {
glog.Errorf("preStop hook for container %q failed: %v", name, err)
dm.generateFailedContainerEvent(containerID, pod.Name, kubecontainer.FailedPreStopHook, msg)
}
}()
select {
@@ -1362,6 +1363,15 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co
return err
}
func (dm *DockerManager) generateFailedContainerEvent(containerID kubecontainer.ContainerID, podName, reason, message string) {
ref, ok := dm.containerRefManager.GetRef(containerID)
if !ok {
glog.Warningf("No ref for pod '%q'", podName)
return
}
dm.recorder.Event(ref, api.EventTypeWarning, reason, message)
}
var errNoPodOnContainer = fmt.Errorf("no pod information labels on Docker container")
// containerAndPodFromLabels tries to load the appropriate container info off of a Docker container's labels
@@ -1477,9 +1487,10 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe
}
if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {
handlerErr := dm.runner.Run(id, pod, container, container.Lifecycle.PostStart)
msg, handlerErr := dm.runner.Run(id, pod, container, container.Lifecycle.PostStart)
if handlerErr != nil {
err := fmt.Errorf("PostStart handler: %v", handlerErr)
dm.generateFailedContainerEvent(id, pod.Name, kubecontainer.FailedPostStartHook, msg)
dm.KillContainerInPod(id, container, pod, err.Error(), nil)
return kubecontainer.ContainerID{}, err
}

8
pkg/kubelet/lifecycle/fake_handler_runner.go
View File

@@ -35,12 +35,12 @@ func NewFakeHandlerRunner() *FakeHandlerRunner {
return &FakeHandlerRunner{HandlerRuns: []string{}}
}
func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error {
func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) {
hr.Lock()
defer hr.Unlock()
if hr.Err != nil {
return hr.Err
return "", hr.Err
}
switch {
@@ -51,9 +51,9 @@ func (hr *FakeHandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api
case handler.TCPSocket != nil:
hr.HandlerRuns = append(hr.HandlerRuns, fmt.Sprintf("tcp-socket on pod: %v, container: %v: %v", format.Pod(pod), container.Name, containerID.String()))
default:
return fmt.Errorf("Invalid handler: %v", handler)
return "", fmt.Errorf("Invalid handler: %v", handler)
}
return nil
return "", nil
}
func (hr *FakeHandlerRunner) Reset() {

22
pkg/kubelet/lifecycle/handlers.go
View File

@@ -52,26 +52,32 @@ func NewHandlerRunner(httpGetter kubetypes.HttpGetter, commandRunner kubecontain
}
}
func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) error {
func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) {
switch {
case handler.Exec != nil:
var buffer bytes.Buffer
var (
buffer bytes.Buffer
msg string
)
output := ioutils.WriteCloserWrapper(&buffer)
err := hr.commandRunner.ExecInContainer(containerID, handler.Exec.Command, nil, output, output, false)
if err != nil {
glog.V(1).Infof("Exec lifecycle hook (%v) for Container %q in Pod %q failed - %q", handler.Exec.Command, container.Name, format.Pod(pod), buffer.String())
msg := fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - %q", handler.Exec.Command, container.Name, format.Pod(pod), buffer.String())
glog.V(1).Infof(msg)
}
return err
return msg, err
case handler.HTTPGet != nil:
msg, err := hr.runHTTPHandler(pod, container, handler)
if err != nil {
glog.V(1).Infof("Http lifecycle hook (%s) for Container %q in Pod %q failed - %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), msg)
msg := fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), msg)
glog.V(1).Infof(msg)
}
return err
return msg, err
default:
err := fmt.Errorf("Invalid handler: %v", handler)
glog.Errorf("Cannot run handler: %v", err)
return err
msg := fmt.Sprintf("Cannot run handler: %v", err)
glog.Errorf(msg)
return msg, err
}
}

28
pkg/kubelet/lifecycle/handlers_test.go
View File

@@ -19,8 +19,10 @@ package lifecycle
import (
"fmt"
"io"
"io/ioutil"
"net/http"
"reflect"
"strings"
"testing"
"k8s.io/kubernetes/pkg/api"
@@ -110,7 +112,7 @@ func TestRunHandlerExec(t *testing.T) {
pod.ObjectMeta.Name = "podFoo"
pod.ObjectMeta.Namespace = "nsFoo"
pod.Spec.Containers = []api.Container{container}
err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
_, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@@ -123,11 +125,12 @@ func TestRunHandlerExec(t *testing.T) {
type fakeHTTP struct {
url string
err error
resp *http.Response
}
func (f *fakeHTTP) Get(url string) (*http.Response, error) {
f.url = url
return nil, f.err
return f.resp, f.err
}
func TestRunHandlerHttp(t *testing.T) {
@@ -153,7 +156,7 @@ func TestRunHandlerHttp(t *testing.T) {
pod.ObjectMeta.Name = "podFoo"
pod.ObjectMeta.Namespace = "nsFoo"
pod.Spec.Containers = []api.Container{container}
err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
_, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
if err != nil {
t.Errorf("unexpected error: %v", err)
@@ -180,7 +183,7 @@ func TestRunHandlerNil(t *testing.T) {
pod.ObjectMeta.Name = podName
pod.ObjectMeta.Namespace = podNamespace
pod.Spec.Containers = []api.Container{container}
err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
_, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
if err == nil {
t.Errorf("expect error, but got nil")
}
@@ -188,14 +191,13 @@ func TestRunHandlerNil(t *testing.T) {
func TestRunHandlerHttpFailure(t *testing.T) {
expectedErr := fmt.Errorf("fake http error")
fakeHttp := fakeHTTP{err: expectedErr}
handlerRunner := &HandlerRunner{
httpGetter: &fakeHttp,
commandRunner: &fakeContainerCommandRunner{},
containerManager: nil,
expectedResp := http.Response{
Body: ioutil.NopCloser(strings.NewReader(expectedErr.Error())),
}
fakeHttp := fakeHTTP{err: expectedErr, resp: &expectedResp}
handlerRunner := NewHandlerRunner(&fakeHttp, &fakeContainerCommandRunner{}, nil)
containerName := "containerFoo"
containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
container := api.Container{
Name: containerName,
Lifecycle: &api.Lifecycle{
@@ -212,12 +214,12 @@ func TestRunHandlerHttpFailure(t *testing.T) {
pod.ObjectMeta.Name = "podFoo"
pod.ObjectMeta.Namespace = "nsFoo"
pod.Spec.Containers = []api.Container{container}
msg, err := handlerRunner.runHTTPHandler(&pod, &container, container.Lifecycle.PostStart)
msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
if err == nil {
t.Errorf("expected error: %v", expectedErr)
}
if msg != "" {
t.Errorf("expected empty error message")
if msg != expectedErr.Error() {
t.Errorf("unexpected error message: %q; expected %q", msg, expectedErr)
}
if fakeHttp.url != "http://foo:8080/bar" {
t.Errorf("unexpected url: %s", fakeHttp.url)

22
pkg/kubelet/rkt/rkt.go
View File

@@ -1184,7 +1184,16 @@ func (r *Runtime) RunPod(pod *api.Pod, pullSecrets []api.Secret) error {
func (r *Runtime) runPreStopHook(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container) error {
glog.V(4).Infof("rkt: Running pre-stop hook for container %q of pod %q", container.Name, format.Pod(pod))
return r.runner.Run(containerID, pod, container, container.Lifecycle.PreStop)
msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PreStop)
if err != nil {
ref, ok := r.containerRefManager.GetRef(containerID)
if !ok {
glog.Warningf("No ref for container %q", containerID)
} else {
r.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedPreStopHook, msg)
}
}
return err
}
func (r *Runtime) runPostStartHook(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container) error {
@@ -1215,7 +1224,16 @@ func (r *Runtime) runPostStartHook(containerID kubecontainer.ContainerID, pod *a
return fmt.Errorf("rkt: Pod %q doesn't become running in %v: %v", format.Pod(pod), timeout, err)
}
return r.runner.Run(containerID, pod, container, container.Lifecycle.PostStart)
msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PostStart)
if err != nil {
ref, ok := r.containerRefManager.GetRef(containerID)
if !ok {
glog.Warningf("No ref for container %q", containerID)
} else {
r.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedPostStartHook, msg)
}
}
return err
}
type lifecycleHookType string