Merge pull request #34814 from Crassirostris/gcl-test-flakiness

Automatic merge from submit-queue

Introduce additional debug logging to cluster logging tests

@piosz

test/e2e/cluster_logging_es.go

@@ -72,6 +72,12 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
 			}
 		}
 
+		if totalMissing > 0 {
+			if err := reportLogsFromFluentdPod(f); err != nil {
+				framework.Logf("Failed to report logs from fluentd pod due to %v", err)
+			}
+		}
+
 		Expect(totalMissing).To(Equal(0), "Some log lines are still missing")
 	})
 })

test/e2e/cluster_logging_gcl.go

@@ -63,6 +63,12 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func()
 			}
 		}
 
+		if totalMissing > 0 {
+			if err := reportLogsFromFluentdPod(f); err != nil {
+				framework.Logf("Failed to report logs from fluentd pod due to %v", err)
+			}
+		}
+
 		Expect(totalMissing).To(Equal(0), "Some log lines are still missing")
 	})
 })

test/e2e/cluster_logging_utils.go

@@ -17,10 +17,12 @@ limitations under the License.
 package e2e
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/labels"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -57,3 +59,33 @@ func createSynthLogger(f *framework.Framework, linesCount int) {
 		},
 	})
 }
+
+func reportLogsFromFluentdPod(f *framework.Framework) error {
+	synthLoggerPod, err := f.PodClient().Get(synthLoggerPodName)
+	if err != nil {
+		return fmt.Errorf("Failed to get synth logger pod due to %v", err)
+	}
+
+	synthLoggerNodeName := synthLoggerPod.Spec.NodeName
+	if synthLoggerNodeName == "" {
+		return errors.New("Synthlogger pod is not assigned to the node")
+	}
+
+	label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "fluentd-logging"}))
+	options := api.ListOptions{LabelSelector: label}
+	fluentdPods, err := f.Client.Pods(api.NamespaceSystem).List(options)
+
+	for _, fluentdPod := range fluentdPods.Items {
+		if fluentdPod.Spec.NodeName == synthLoggerNodeName {
+			containerName := fluentdPod.Spec.Containers[0].Name
+			logs, err := framework.GetPodLogs(f.Client, api.NamespaceSystem, fluentdPod.Name, containerName)
+			if err != nil {
+				return fmt.Errorf("Failed to get logs from fluentd pod %s due to %v", fluentdPod.Name, err)
+			}
+			framework.Logf("Logs from fluentd pod %s:\n%s", fluentdPod.Name, logs)
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Failed to find fluentd pod running on node %s", synthLoggerNodeName)
+}