Allow panics and unhandled errors to be reported to external targets

It would be better if Kubernetes could be instrumented to report panics
and handled but unreturned errors in controller loops to a remote
service.  This commit introduces settable handlers for HandleCrash and a new
HandleError that can be overriden to report errors to a remote service
for analysis.  HandleError() in particular is for control loops that do
not return an error, and so there is no ability to report those errors
other than in log files.

ReplicationController is provided as an example.

pkg/util/util_test.go

@@ -18,6 +18,7 @@ package util
 
 import (
 	"encoding/json"
+	"fmt"
 	"reflect"
 	"testing"
 
@@ -59,6 +60,40 @@ func TestHandleCrash(t *testing.T) {
 	}
 }
 
+func TestCustomHandleCrash(t *testing.T) {
+	old := PanicHandlers
+	defer func() { PanicHandlers = old }()
+	var result interface{}
+	PanicHandlers = []func(interface{}){
+		func(r interface{}) {
+			result = r
+		},
+	}
+	func() {
+		defer HandleCrash()
+		panic("test")
+	}()
+	if result != "test" {
+		t.Errorf("did not receive custom handler")
+	}
+}
+
+func TestCustomHandleError(t *testing.T) {
+	old := ErrorHandlers
+	defer func() { ErrorHandlers = old }()
+	var result error
+	ErrorHandlers = []func(error){
+		func(err error) {
+			result = err
+		},
+	}
+	err := fmt.Errorf("test")
+	HandleError(err)
+	if result != err {
+		t.Errorf("did not receive custom handler")
+	}
+}
+
 func TestNewIntOrStringFromInt(t *testing.T) {
 	i := NewIntOrStringFromInt(93)
 	if i.Kind != IntstrInt || i.IntVal != 93 {