diff --git a/test/e2e/cluster_logging_es.go b/test/e2e/cluster_logging_es.go index 2ce495ac28e..339a72c579b 100644 --- a/test/e2e/cluster_logging_es.go +++ b/test/e2e/cluster_logging_es.go @@ -72,6 +72,12 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu } } + if totalMissing > 0 { + if err := reportLogsFromFluentdPod(f); err != nil { + framework.Logf("Failed to report logs from fluentd pod due to %v", err) + } + } + Expect(totalMissing).To(Equal(0), "Some log lines are still missing") }) }) diff --git a/test/e2e/cluster_logging_gcl.go b/test/e2e/cluster_logging_gcl.go index 0cf5fe0143c..3215c67b013 100644 --- a/test/e2e/cluster_logging_gcl.go +++ b/test/e2e/cluster_logging_gcl.go @@ -63,6 +63,12 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func() } } + if totalMissing > 0 { + if err := reportLogsFromFluentdPod(f); err != nil { + framework.Logf("Failed to report logs from fluentd pod due to %v", err) + } + } + Expect(totalMissing).To(Equal(0), "Some log lines are still missing") }) }) diff --git a/test/e2e/cluster_logging_utils.go b/test/e2e/cluster_logging_utils.go index aa5ba610790..ce1f354b8a0 100644 --- a/test/e2e/cluster_logging_utils.go +++ b/test/e2e/cluster_logging_utils.go @@ -17,10 +17,12 @@ limitations under the License. package e2e import ( + "errors" "fmt" "time" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/test/e2e/framework" ) @@ -57,3 +59,33 @@ func createSynthLogger(f *framework.Framework, linesCount int) { }, }) } + +func reportLogsFromFluentdPod(f *framework.Framework) error { + synthLoggerPod, err := f.PodClient().Get(synthLoggerPodName) + if err != nil { + return fmt.Errorf("Failed to get synth logger pod due to %v", err) + } + + synthLoggerNodeName := synthLoggerPod.Spec.NodeName + if synthLoggerNodeName == "" { + return errors.New("Synthlogger pod is not assigned to the node") + } + + label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "fluentd-logging"})) + options := api.ListOptions{LabelSelector: label} + fluentdPods, err := f.Client.Pods(api.NamespaceSystem).List(options) + + for _, fluentdPod := range fluentdPods.Items { + if fluentdPod.Spec.NodeName == synthLoggerNodeName { + containerName := fluentdPod.Spec.Containers[0].Name + logs, err := framework.GetPodLogs(f.Client, api.NamespaceSystem, fluentdPod.Name, containerName) + if err != nil { + return fmt.Errorf("Failed to get logs from fluentd pod %s due to %v", fluentdPod.Name, err) + } + framework.Logf("Logs from fluentd pod %s:\n%s", fluentdPod.Name, logs) + return nil + } + } + + return fmt.Errorf("Failed to find fluentd pod running on node %s", synthLoggerNodeName) +}