github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support extended pod logging options

Browse Source 
Increase the supported controls on pod logging. Add validaiton to pod
log options. Ensure the Kubelet is using a consistent, structured way to
process pod log arguments.

Add ?sinceSeconds=<durationInSeconds>, &sinceTime=<RFC3339>, ?timestamps=<bool>,
?tailLines=<number>, and ?limitBytes=<number>
This commit is contained in:
Clayton Coleman
2015-09-09 23:46:11 -04:00
parent a02bcefa10
commit c2e90cd154
30 changed files with 734 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/kubelet/container/fake_runtime.go
View File

@@ -251,7 +251,7 @@ func (f *FakeRuntime) RunInContainer(containerID string, cmd []string) ([]byte,
return []byte{}, f.Err
}
func (f *FakeRuntime) GetContainerLogs(pod *api.Pod, containerID, tail string, follow bool, stdout, stderr io.Writer) (err error) {
func (f *FakeRuntime) GetContainerLogs(pod *api.Pod, containerID string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) (err error) {
f.Lock()
defer f.Unlock()

2
pkg/kubelet/container/runtime.go
View File

@@ -79,7 +79,7 @@ type Runtime interface {
// default, it returns a snapshot of the container log. Set 'follow' to true to
// stream the log. Set 'follow' to false and specify the number of lines (e.g.
// "100" or "all") to tail the log.
GetContainerLogs(pod *api.Pod, containerID, tail string, follow bool, stdout, stderr io.Writer) (err error)
GetContainerLogs(pod *api.Pod, containerID string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) (err error)
// ContainerCommandRunner encapsulates the command runner interfaces for testability.
ContainerCommandRunner
// ContainerAttach encapsulates the attaching to containers for testability

19
pkg/kubelet/dockertools/manager.go
View File

@@ -259,20 +259,29 @@ func (sc *reasonInfoCache) Get(uid types.UID, name string) (reasonInfo, bool) {
// stream the log. Set 'follow' to false and specify the number of lines (e.g.
// "100" or "all") to tail the log.
// TODO: Make 'RawTerminal' option flagable.
func (dm *DockerManager) GetContainerLogs(pod *api.Pod, containerID, tail string, follow bool, stdout, stderr io.Writer) (err error) {
func (dm *DockerManager) GetContainerLogs(pod *api.Pod, containerID string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) (err error) {
var since int64
if logOptions.SinceSeconds != nil {
t := unversioned.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second)
since = t.Unix()
}
if logOptions.SinceTime != nil {
since = logOptions.SinceTime.Unix()
}
opts := docker.LogsOptions{
Container: containerID,
Stdout: true,
Stderr: true,
OutputStream: stdout,
ErrorStream: stderr,
Timestamps: false,
Timestamps: logOptions.Timestamps,
Since: since,
Follow: logOptions.Follow,
RawTerminal: false,
Follow: follow,
}
if !follow {
opts.Tail = tail
if !logOptions.Follow && logOptions.TailLines != nil {
opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10)
}
err = dm.client.Logs(opts)

6
pkg/kubelet/kubelet.go
View File

@@ -1916,7 +1916,7 @@ func (kl *Kubelet) validateContainerStatus(podStatus *api.PodStatus, containerNa
// GetKubeletContainerLogs returns logs from the container
// TODO: this method is returning logs of random container attempts, when it should be returning the most recent attempt
// or all of them.
func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName, tail string, follow, previous bool, stdout, stderr io.Writer) error {
func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error {
// TODO(vmarmol): Refactor to not need the pod status and verification.
// Pod workers periodically write status to statusManager. If status is not
// cached there, something is wrong (or kubelet just restarted and hasn't
@@ -1940,13 +1940,13 @@ func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName, tail stri
// No log is available if pod is not in a "known" phase (e.g. Unknown).
return fmt.Errorf("Pod %q in namespace %q : %v", name, namespace, err)
}
containerID, err := kl.validateContainerStatus(&podStatus, containerName, previous)
containerID, err := kl.validateContainerStatus(&podStatus, containerName, logOptions.Previous)
if err != nil {
// No log is available if the container status is missing or is in the
// waiting state.
return fmt.Errorf("Pod %q in namespace %q: %v", name, namespace, err)
}
return kl.containerRuntime.GetContainerLogs(pod, containerID, tail, follow, stdout, stderr)
return kl.containerRuntime.GetContainerLogs(pod, containerID, logOptions, stdout, stderr)
}
// GetHostname Returns the hostname as the kubelet sees it.

11
pkg/kubelet/rkt/rkt.go
View File

@@ -1054,23 +1054,20 @@ func (r *runtime) SyncPod(pod *api.Pod, runningPod kubecontainer.Pod, podStatus
// See https://github.com/coreos/rkt/blob/master/Documentation/commands.md#logging for more details.
//
// TODO(yifan): If the rkt is using lkvm as the stage1 image, then this function will fail.
func (r *runtime) GetContainerLogs(pod *api.Pod, containerID string, tail string, follow bool, stdout, stderr io.Writer) error {
func (r *runtime) GetContainerLogs(pod *api.Pod, containerID string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error {
id, err := parseContainerID(containerID)
if err != nil {
return err
}
cmd := exec.Command("journalctl", "-M", fmt.Sprintf("rkt-%s", id.uuid), "-u", id.appName)
if follow {
if logOptions.Follow {
cmd.Args = append(cmd.Args, "-f")
}
if tail == "all" {
if logOptions.TailLines == nil {
cmd.Args = append(cmd.Args, "-a")
} else {
_, err := strconv.Atoi(tail)
if err == nil {
cmd.Args = append(cmd.Args, "-n", tail)
}
cmd.Args = append(cmd.Args, "-n", strconv.FormatInt(*logOptions.TailLines, 10))
}
cmd.Stdout, cmd.Stderr = stdout, stderr
return cmd.Run()

47
pkg/kubelet/server.go
View File

@@ -36,7 +36,11 @@ import (
cadvisorApi "github.com/google/cadvisor/info/v1"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/kubernetes/pkg/api"
apierrs "k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/api/latest"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/api/validation"
"k8s.io/kubernetes/pkg/healthz"
"k8s.io/kubernetes/pkg/httplog"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -44,6 +48,7 @@ import (
"k8s.io/kubernetes/pkg/util/flushwriter"
"k8s.io/kubernetes/pkg/util/httpstream"
"k8s.io/kubernetes/pkg/util/httpstream/spdy"
"k8s.io/kubernetes/pkg/util/limitwriter"
)
// Server is a http.Handler which exposes kubelet functionality over HTTP.
@@ -102,7 +107,7 @@ type HostInterface interface {
RunInContainer(name string, uid types.UID, container string, cmd []string) ([]byte, error)
ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool) error
AttachContainer(name string, uid types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool) error
GetKubeletContainerLogs(podFullName, containerName, tail string, follow, previous bool, stdout, stderr io.Writer) error
GetKubeletContainerLogs(podFullName, containerName string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error
ServeLogs(w http.ResponseWriter, req *http.Request)
PortForward(name string, uid types.UID, port uint16, stream io.ReadWriteCloser) error
StreamingConnectionIdleTimeout() time.Duration
@@ -308,6 +313,7 @@ func (s *Server) getContainerLogs(request *restful.Request, response *restful.Re
if len(podID) == 0 {
// TODO: Why return JSON when the rest return plaintext errors?
// TODO: Why return plaintext errors?
response.WriteError(http.StatusBadRequest, fmt.Errorf(`{"message": "Missing podID."}`))
return
}
@@ -322,9 +328,32 @@ func (s *Server) getContainerLogs(request *restful.Request, response *restful.Re
return
}
follow, _ := strconv.ParseBool(request.QueryParameter("follow"))
previous, _ := strconv.ParseBool(request.QueryParameter("previous"))
tail := request.QueryParameter("tail")
query := request.Request.URL.Query()
// backwards compatibility for the "tail" query parameter
if tail := request.QueryParameter("tail"); len(tail) > 0 {
query["tailLines"] = []string{tail}
// "all" is the same as omitting tail
if tail == "all" {
delete(query, "tailLines")
}
}
// container logs on the kubelet are locked to v1
versioned := &v1.PodLogOptions{}
if err := api.Scheme.Convert(&query, versioned); err != nil {
response.WriteError(http.StatusBadRequest, fmt.Errorf(`{"message": "Unable to decode query."}`))
return
}
out, err := api.Scheme.ConvertToVersion(versioned, "")
if err != nil {
response.WriteError(http.StatusBadRequest, fmt.Errorf(`{"message": "Unable to convert request query."}`))
return
}
logOptions := out.(*api.PodLogOptions)
logOptions.TypeMeta = unversioned.TypeMeta{}
if errs := validation.ValidatePodLogOptions(logOptions); len(errs) > 0 {
response.WriteError(apierrs.StatusUnprocessableEntity, fmt.Errorf(`{"message": "Invalid request."}`))
return
}
pod, ok := s.host.GetPodByName(podNamespace, podID)
if !ok {
@@ -348,11 +377,15 @@ func (s *Server) getContainerLogs(request *restful.Request, response *restful.Re
return
}
fw := flushwriter.Wrap(response.ResponseWriter)
if logOptions.LimitBytes != nil {
fw = limitwriter.New(fw, *logOptions.LimitBytes)
}
response.Header().Set("Transfer-Encoding", "chunked")
response.WriteHeader(http.StatusOK)
err := s.host.GetKubeletContainerLogs(kubecontainer.GetPodFullName(pod), containerName, tail, follow, previous, fw, fw)
if err != nil {
response.WriteError(http.StatusInternalServerError, err)
if err := s.host.GetKubeletContainerLogs(kubecontainer.GetPodFullName(pod), containerName, logOptions, fw, fw); err != nil {
if err != limitwriter.ErrMaximumWrite {
response.WriteError(http.StatusInternalServerError, err)
}
return
}
}

134
pkg/kubelet/server_test.go
View File

@@ -34,6 +34,7 @@ import (
cadvisorApi "github.com/google/cadvisor/info/v1"
"k8s.io/kubernetes/pkg/api"
apierrs "k8s.io/kubernetes/pkg/api/errors"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/types"
@@ -54,7 +55,7 @@ type fakeKubelet struct {
execFunc func(pod string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool) error
attachFunc func(pod string, uid types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool) error
portForwardFunc func(name string, uid types.UID, port uint16, stream io.ReadWriteCloser) error
containerLogsFunc func(podFullName, containerName, tail string, follow, pervious bool, stdout, stderr io.Writer) error
containerLogsFunc func(podFullName, containerName string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error
streamingConnectionIdleTimeoutFunc func() time.Duration
hostnameFunc func() string
resyncInterval time.Duration
@@ -101,8 +102,8 @@ func (fk *fakeKubelet) ServeLogs(w http.ResponseWriter, req *http.Request) {
fk.logFunc(w, req)
}
func (fk *fakeKubelet) GetKubeletContainerLogs(podFullName, containerName, tail string, follow, previous bool, stdout, stderr io.Writer) error {
return fk.containerLogsFunc(podFullName, containerName, tail, follow, previous, stdout, stderr)
func (fk *fakeKubelet) GetKubeletContainerLogs(podFullName, containerName string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error {
return fk.containerLogsFunc(podFullName, containerName, logOptions, stdout, stderr)
}
func (fk *fakeKubelet) GetHostname() string {
@@ -558,22 +559,16 @@ func setPodByNameFunc(fw *serverTestFramework, namespace, pod, container string)
}
}
func setGetContainerLogsFunc(fw *serverTestFramework, t *testing.T, expectedPodName, expectedContainerName, expectedTail string, expectedFollow, expectedPrevious bool, output string) {
fw.fakeKubelet.containerLogsFunc = func(podFullName, containerName, tail string, follow, previous bool, stdout, stderr io.Writer) error {
func setGetContainerLogsFunc(fw *serverTestFramework, t *testing.T, expectedPodName, expectedContainerName string, expectedLogOptions *api.PodLogOptions, output string) {
fw.fakeKubelet.containerLogsFunc = func(podFullName, containerName string, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error {
if podFullName != expectedPodName {
t.Errorf("expected %s, got %s", expectedPodName, podFullName)
}
if containerName != expectedContainerName {
t.Errorf("expected %s, got %s", expectedContainerName, containerName)
}
if tail != expectedTail {
t.Errorf("expected %s, got %s", expectedTail, tail)
}
if follow != expectedFollow {
t.Errorf("expected %t, got %t", expectedFollow, follow)
}
if previous != expectedPrevious {
t.Errorf("expected %t, got %t", expectedPrevious, previous)
if !reflect.DeepEqual(expectedLogOptions, logOptions) {
t.Errorf("expected %#v, got %#v", expectedLogOptions, logOptions)
}
io.WriteString(stdout, output)
@@ -581,6 +576,7 @@ func setGetContainerLogsFunc(fw *serverTestFramework, t *testing.T, expectedPodN
}
}
// TODO: I really want to be a table driven test
func TestContainerLogs(t *testing.T) {
fw := newServerTest()
output := "foo bar"
@@ -588,11 +584,8 @@ func TestContainerLogs(t *testing.T) {
podName := "foo"
expectedPodName := getPodName(podName, podNamespace)
expectedContainerName := "baz"
expectedTail := ""
expectedFollow := false
expectedPrevious := false
setPodByNameFunc(fw, podNamespace, podName, expectedContainerName)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, expectedTail, expectedFollow, expectedPrevious, output)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, &api.PodLogOptions{}, output)
resp, err := http.Get(fw.testHTTPServer.URL + "/containerLogs/" + podNamespace + "/" + podName + "/" + expectedContainerName)
if err != nil {
t.Errorf("Got error GETing: %v", err)
@@ -609,6 +602,32 @@ func TestContainerLogs(t *testing.T) {
}
}
func TestContainerLogsWithLimitBytes(t *testing.T) {
fw := newServerTest()
output := "foo bar"
podNamespace := "other"
podName := "foo"
expectedPodName := getPodName(podName, podNamespace)
expectedContainerName := "baz"
bytes := int64(3)
setPodByNameFunc(fw, podNamespace, podName, expectedContainerName)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, &api.PodLogOptions{LimitBytes: &bytes}, output)
resp, err := http.Get(fw.testHTTPServer.URL + "/containerLogs/" + podNamespace + "/" + podName + "/" + expectedContainerName + "?limitBytes=3")
if err != nil {
t.Errorf("Got error GETing: %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("Error reading container logs: %v", err)
}
result := string(body)
if result != output[:bytes] {
t.Errorf("Expected: '%v', got: '%v'", output[:bytes], result)
}
}
func TestContainerLogsWithTail(t *testing.T) {
fw := newServerTest()
output := "foo bar"
@@ -616,11 +635,35 @@ func TestContainerLogsWithTail(t *testing.T) {
podName := "foo"
expectedPodName := getPodName(podName, podNamespace)
expectedContainerName := "baz"
expectedTail := "5"
expectedFollow := false
expectedPrevious := false
expectedTail := int64(5)
setPodByNameFunc(fw, podNamespace, podName, expectedContainerName)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, expectedTail, expectedFollow, expectedPrevious, output)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, &api.PodLogOptions{TailLines: &expectedTail}, output)
resp, err := http.Get(fw.testHTTPServer.URL + "/containerLogs/" + podNamespace + "/" + podName + "/" + expectedContainerName + "?tailLines=5")
if err != nil {
t.Errorf("Got error GETing: %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("Error reading container logs: %v", err)
}
result := string(body)
if result != output {
t.Errorf("Expected: '%v', got: '%v'", output, result)
}
}
func TestContainerLogsWithLegacyTail(t *testing.T) {
fw := newServerTest()
output := "foo bar"
podNamespace := "other"
podName := "foo"
expectedPodName := getPodName(podName, podNamespace)
expectedContainerName := "baz"
expectedTail := int64(5)
setPodByNameFunc(fw, podNamespace, podName, expectedContainerName)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, &api.PodLogOptions{TailLines: &expectedTail}, output)
resp, err := http.Get(fw.testHTTPServer.URL + "/containerLogs/" + podNamespace + "/" + podName + "/" + expectedContainerName + "?tail=5")
if err != nil {
t.Errorf("Got error GETing: %v", err)
@@ -637,6 +680,50 @@ func TestContainerLogsWithTail(t *testing.T) {
}
}
func TestContainerLogsWithTailAll(t *testing.T) {
fw := newServerTest()
output := "foo bar"
podNamespace := "other"
podName := "foo"
expectedPodName := getPodName(podName, podNamespace)
expectedContainerName := "baz"
setPodByNameFunc(fw, podNamespace, podName, expectedContainerName)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, &api.PodLogOptions{}, output)
resp, err := http.Get(fw.testHTTPServer.URL + "/containerLogs/" + podNamespace + "/" + podName + "/" + expectedContainerName + "?tail=all")
if err != nil {
t.Errorf("Got error GETing: %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("Error reading container logs: %v", err)
}
result := string(body)
if result != output {
t.Errorf("Expected: '%v', got: '%v'", output, result)
}
}
func TestContainerLogsWithInvalidTail(t *testing.T) {
fw := newServerTest()
output := "foo bar"
podNamespace := "other"
podName := "foo"
expectedPodName := getPodName(podName, podNamespace)
expectedContainerName := "baz"
setPodByNameFunc(fw, podNamespace, podName, expectedContainerName)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, &api.PodLogOptions{}, output)
resp, err := http.Get(fw.testHTTPServer.URL + "/containerLogs/" + podNamespace + "/" + podName + "/" + expectedContainerName + "?tail=-1")
if err != nil {
t.Errorf("Got error GETing: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != apierrs.StatusUnprocessableEntity {
t.Errorf("Unexpected non-error reading container logs: %#v", resp)
}
}
func TestContainerLogsWithFollow(t *testing.T) {
fw := newServerTest()
output := "foo bar"
@@ -644,11 +731,8 @@ func TestContainerLogsWithFollow(t *testing.T) {
podName := "foo"
expectedPodName := getPodName(podName, podNamespace)
expectedContainerName := "baz"
expectedTail := ""
expectedFollow := true
expectedPrevious := false
setPodByNameFunc(fw, podNamespace, podName, expectedContainerName)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, expectedTail, expectedFollow, expectedPrevious, output)
setGetContainerLogsFunc(fw, t, expectedPodName, expectedContainerName, &api.PodLogOptions{Follow: true}, output)
resp, err := http.Get(fw.testHTTPServer.URL + "/containerLogs/" + podNamespace + "/" + podName + "/" + expectedContainerName + "?follow=1")
if err != nil {
t.Errorf("Got error GETing: %v", err)