196 lines
7.2 KiB
Go
196 lines
7.2 KiB
Go
/*
|
|
Copyright 2017 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package stackdriver
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"k8s.io/apimachinery/pkg/util/json"
|
|
"k8s.io/apimachinery/pkg/util/uuid"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
|
instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
|
|
"k8s.io/kubernetes/test/e2e/instrumentation/logging/utils"
|
|
|
|
"github.com/onsi/ginkgo"
|
|
)
|
|
|
|
const (
|
|
ingestionInterval = 10 * time.Second
|
|
ingestionTimeout = 10 * time.Minute
|
|
)
|
|
|
|
var _ = instrumentation.SIGDescribe("Cluster level logging implemented by Stackdriver", func() {
|
|
f := framework.NewDefaultFramework("sd-logging")
|
|
|
|
ginkgo.BeforeEach(func() {
|
|
e2eskipper.SkipUnlessProviderIs("gce", "gke")
|
|
})
|
|
|
|
ginkgo.It("should ingest logs [Feature:StackdriverLogging]", func() {
|
|
withLogProviderForScope(f, podsScope, func(p *sdLogProvider) {
|
|
ginkgo.By("Checking ingesting text logs", func() {
|
|
pod, err := utils.StartAndReturnSelf(utils.NewRepeatingLoggingPod("synthlogger-1", "hey"), f)
|
|
framework.ExpectNoError(err, "Failed to start a pod")
|
|
|
|
ginkgo.By("Waiting for logs to ingest")
|
|
c := utils.NewLogChecker(p, utils.UntilFirstEntry, utils.JustTimeout, pod.Name())
|
|
err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
|
|
framework.ExpectNoError(err)
|
|
})
|
|
|
|
ginkgo.By("Checking ingesting json logs", func() {
|
|
logRaw := "{\"a\":\"b\"}"
|
|
pod, err := utils.StartAndReturnSelf(utils.NewRepeatingLoggingPod("synthlogger-2", logRaw), f)
|
|
framework.ExpectNoError(err, "Failed to start a pod")
|
|
|
|
ginkgo.By("Waiting for logs to ingest")
|
|
c := utils.NewLogChecker(p, func(_ string, logEntries []utils.LogEntry) (bool, error) {
|
|
if len(logEntries) == 0 {
|
|
return false, nil
|
|
}
|
|
log := logEntries[0]
|
|
if log.JSONPayload == nil {
|
|
return false, fmt.Errorf("log entry unexpectedly is not json: %s", log.TextPayload)
|
|
}
|
|
if log.JSONPayload["a"] != "b" {
|
|
bytes, err := json.Marshal(log.JSONPayload)
|
|
if err != nil {
|
|
return false, fmt.Errorf("log entry ingested incorrectly, failed to marshal: %v", err)
|
|
}
|
|
return false, fmt.Errorf("log entry ingested incorrectly, got %v, want %s",
|
|
string(bytes), logRaw)
|
|
}
|
|
return true, nil
|
|
}, utils.JustTimeout, pod.Name())
|
|
err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
|
|
framework.ExpectNoError(err)
|
|
})
|
|
|
|
ginkgo.By("Checking ingesting logs in glog format", func() {
|
|
logUnformatted := "Text"
|
|
logRaw := fmt.Sprintf("I0101 00:00:00.000000 1 main.go:1] %s", logUnformatted)
|
|
pod, err := utils.StartAndReturnSelf(utils.NewRepeatingLoggingPod("synthlogger-3", logRaw), f)
|
|
framework.ExpectNoError(err, "Failed to start a pod")
|
|
|
|
ginkgo.By("Waiting for logs to ingest")
|
|
c := utils.NewLogChecker(p, func(_ string, logEntries []utils.LogEntry) (bool, error) {
|
|
if len(logEntries) == 0 {
|
|
return false, nil
|
|
}
|
|
log := logEntries[0]
|
|
if log.TextPayload == "" {
|
|
return false, fmt.Errorf("log entry is unexpectedly json: %v", log.JSONPayload)
|
|
}
|
|
if log.TextPayload != logUnformatted {
|
|
return false, fmt.Errorf("log entry ingested incorrectly, got %s, want %s",
|
|
log.TextPayload, logUnformatted)
|
|
}
|
|
return true, nil
|
|
}, utils.JustTimeout, pod.Name())
|
|
err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
|
|
framework.ExpectNoError(err)
|
|
})
|
|
|
|
ginkgo.By("Checking that too long lines are trimmed", func() {
|
|
maxLength := 100 * 1024
|
|
cmd := []string{
|
|
"/bin/sh",
|
|
"-c",
|
|
fmt.Sprintf("while :; do printf '%%*s' %d | tr ' ' 'A'; echo; sleep 60; done", maxLength+1),
|
|
}
|
|
|
|
pod, err := utils.StartAndReturnSelf(utils.NewExecLoggingPod("synthlogger-4", cmd), f)
|
|
framework.ExpectNoError(err, "Failed to start a pod")
|
|
|
|
ginkgo.By("Waiting for logs to ingest")
|
|
c := utils.NewLogChecker(p, func(_ string, logEntries []utils.LogEntry) (bool, error) {
|
|
if len(logEntries) == 0 {
|
|
return false, nil
|
|
}
|
|
log := logEntries[0]
|
|
if log.JSONPayload != nil {
|
|
return false, fmt.Errorf("got json log entry %v, wanted plain text", log.JSONPayload)
|
|
}
|
|
if len(log.TextPayload) > maxLength {
|
|
return false, fmt.Errorf("got too long entry of length %d", len(log.TextPayload))
|
|
}
|
|
return true, nil
|
|
}, utils.JustTimeout, pod.Name())
|
|
err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
|
|
framework.ExpectNoError(err)
|
|
})
|
|
})
|
|
})
|
|
|
|
ginkgo.It("should ingest events [Feature:StackdriverLogging]", func() {
|
|
eventCreationInterval := 10 * time.Second
|
|
|
|
withLogProviderForScope(f, eventsScope, func(p *sdLogProvider) {
|
|
ginkgo.By("Running pods to generate events while waiting for some of them to be ingested")
|
|
stopCh := make(chan struct{})
|
|
cleanupCh := make(chan struct{})
|
|
defer func() { <-cleanupCh }()
|
|
defer close(stopCh)
|
|
go func() {
|
|
defer ginkgo.GinkgoRecover()
|
|
defer close(cleanupCh)
|
|
|
|
wait.PollUntil(eventCreationInterval, func() (bool, error) {
|
|
podName := fmt.Sprintf("synthlogger-%s", string(uuid.NewUUID()))
|
|
err := utils.NewLoadLoggingPod(podName, "", 1, 1*time.Second).Start(f)
|
|
if err != nil {
|
|
framework.Logf("Failed to create a logging pod: %v", err)
|
|
}
|
|
return false, nil
|
|
}, stopCh)
|
|
}()
|
|
|
|
ginkgo.By("Waiting for events to ingest")
|
|
location := framework.TestContext.CloudConfig.Zone
|
|
if framework.TestContext.CloudConfig.MultiMaster {
|
|
location = framework.TestContext.CloudConfig.Region
|
|
}
|
|
c := utils.NewLogChecker(p, utils.UntilFirstEntryFromLocation(location), utils.JustTimeout, "")
|
|
err := utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
|
|
framework.ExpectNoError(err)
|
|
})
|
|
})
|
|
|
|
ginkgo.It("should ingest system logs from all nodes [Feature:StackdriverLogging]", func() {
|
|
withLogProviderForScope(f, systemScope, func(p *sdLogProvider) {
|
|
ginkgo.By("Waiting for some kubelet logs to be ingested from each node", func() {
|
|
nodeIds := utils.GetNodeIds(f.ClientSet)
|
|
log := fmt.Sprintf("projects/%s/logs/kubelet", framework.TestContext.CloudConfig.ProjectID)
|
|
c := utils.NewLogChecker(p, utils.UntilFirstEntryFromLog(log), utils.JustTimeout, nodeIds...)
|
|
err := utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
|
|
framework.ExpectNoError(err)
|
|
})
|
|
|
|
ginkgo.By("Waiting for some container runtime logs to be ingested from each node", func() {
|
|
nodeIds := utils.GetNodeIds(f.ClientSet)
|
|
log := fmt.Sprintf("projects/%s/logs/container-runtime", framework.TestContext.CloudConfig.ProjectID)
|
|
c := utils.NewLogChecker(p, utils.UntilFirstEntryFromLog(log), utils.JustTimeout, nodeIds...)
|
|
err := utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
|
|
framework.ExpectNoError(err)
|
|
})
|
|
})
|
|
})
|
|
})
|