github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #124645 from matthyx/ms

Browse Source 
e2e lifecycle: reduce test flackiness
This commit is contained in:
Kubernetes Prow Robot
2024-05-01 16:19:55 -07:00
committed by GitHub
parent d387c0c903 f7ea5f3fe1
commit 82cd82aa15
2 changed files with 38 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
test/e2e_node/container_lifecycle_pod_construction.go
View File

@@ -22,7 +22,6 @@ import (
"context" "context"
"fmt" "fmt"
"sort" "sort"
"strconv"
"strings" "strings"
"time" "time"
@@ -62,9 +61,9 @@ func ExecCommand(name string, c execCommand) []string {
// all outputs are in the format of: // all outputs are in the format of:
// time-since-boot timestamp container-name message // time-since-boot timestamp container-name message
// The busybox time command doesn't support sub-second display. uptime displays in hundredths of a second, so we // The busybox time command doesn't support sub-second display.
// include both and use time since boot for relative ordering of file entries // We have to use nmeter to get the milliseconds part.
timeCmd := "`date +%s` `cat /proc/uptime | awk '{print $1}'`" timeCmd := "`date -u +%FT$(nmeter -d0 '%3t' | head -n1)Z`"
containerName := name containerName := name
if c.ContainerName != "" { if c.ContainerName != "" {
containerName = c.ContainerName containerName = c.ContainerName
@@ -107,8 +106,6 @@ func sleepCommand(seconds int) string {
type containerOutput struct { type containerOutput struct {
// time the message was seen to the nearest second // time the message was seen to the nearest second
timestamp time.Time timestamp time.Time
// time the message was seen since the host booted, to the nearest hundredth of a second
timeSinceBoot float64
containerName string containerName string
command string command string
} }
@@ -117,7 +114,7 @@ type containerOutputList []containerOutput
func (o containerOutputList) String() string { func (o containerOutputList) String() string {
var b bytes.Buffer var b bytes.Buffer
for i, v := range o { for i, v := range o {
fmt.Fprintf(&b, "%d) %s %f %s %s\n", i, v.timestamp, v.timeSinceBoot, v.containerName, v.command) fmt.Fprintf(&b, "%d) %s %s %s\n", i, v.timestamp, v.containerName, v.command)
} }
return b.String() return b.String()
} }
@@ -127,8 +124,8 @@ func (o containerOutputList) RunTogether(lhs, rhs string) error {
lhsStart := o.findIndex(lhs, "Started", 0) lhsStart := o.findIndex(lhs, "Started", 0)
rhsStart := o.findIndex(rhs, "Started", 0) rhsStart := o.findIndex(rhs, "Started", 0)
lhsFinish := o.findIndex(lhs, "Finishing", 0) lhsFinish := o.findIndex(lhs, "Exiting", 0)
rhsFinish := o.findIndex(rhs, "Finishing", 0) rhsFinish := o.findIndex(rhs, "Exiting", 0)
if lhsStart == -1 { if lhsStart == -1 {
return fmt.Errorf("couldn't find that %s ever started, got\n%v", lhs, o) return fmt.Errorf("couldn't find that %s ever started, got\n%v", lhs, o)
@@ -138,11 +135,11 @@ func (o containerOutputList) RunTogether(lhs, rhs string) error {
} }
if lhsFinish != -1 && rhsStart > lhsFinish { if lhsFinish != -1 && rhsStart > lhsFinish {
return fmt.Errorf("expected %s to start before finishing %s, got\n%v", rhs, lhs, o) return fmt.Errorf("expected %s to start before exiting %s, got\n%v", rhs, lhs, o)
} }
if rhsFinish != -1 && lhsStart > rhsFinish { if rhsFinish != -1 && lhsStart > rhsFinish {
return fmt.Errorf("expected %s to start before finishing %s, got\n%v", lhs, rhs, o) return fmt.Errorf("expected %s to start before exiting %s, got\n%v", lhs, rhs, o)
} }
return nil return nil
@@ -294,21 +291,22 @@ func (o containerOutputList) findLastIndex(name string, command string) int {
return found return found
} }
// TimeOfStart returns the time since the node boot in floating point seconds that the specified container started. // TimeOfStart returns the UNIX time in milliseconds when the specified container started.
func (o containerOutputList) TimeOfStart(name string) (float64, error) { func (o containerOutputList) TimeOfStart(name string) (int64, error) {
idx := o.findIndex(name, "Starting", 0) idx := o.findIndex(name, "Starting", 0)
if idx == -1 { if idx == -1 {
return 0.0, fmt.Errorf("couldn't find that %s ever started, got\n%v", name, o) return 0, fmt.Errorf("couldn't find that %s ever started, got\n%v", name, o)
} }
return o[idx].timeSinceBoot, nil return o[idx].timestamp.UnixMilli(), nil
} }
func (o containerOutputList) TimeOfLastLoop(name string) (float64, error) { // TimeOfLastLoop returns the UNIX time in milliseconds when the specified container last looped.
func (o containerOutputList) TimeOfLastLoop(name string) (int64, error) {
idx := o.findLastIndex(name, "Looping") idx := o.findLastIndex(name, "Looping")
if idx == -1 { if idx == -1 {
return 0.0, fmt.Errorf("couldn't find that %s ever looped, got\n%v", name, o) return 0, fmt.Errorf("couldn't find that %s ever looped, got\n%v", name, o)
} }
return o[idx].timeSinceBoot, nil return o[idx].timestamp.UnixMilli(), nil
} }
// parseOutput combines the container log from all of the init and regular // parseOutput combines the container log from all of the init and regular
@@ -337,24 +335,21 @@ func parseOutput(ctx context.Context, f *framework.Framework, pod *v1.Pod) conta
var res containerOutputList var res containerOutputList
for sc.Scan() { for sc.Scan() {
fields := strings.Fields(sc.Text()) fields := strings.Fields(sc.Text())
if len(fields) < 4 { if len(fields) < 3 {
framework.ExpectNoError(fmt.Errorf("%v should have at least length 3", fields)) framework.ExpectNoError(fmt.Errorf("%v should have at least length 3", fields))
} }
timestamp, err := strconv.ParseInt(fields[0], 10, 64) timestamp, err := time.Parse(time.RFC3339, fields[0])
framework.ExpectNoError(err)
timeSinceBoot, err := strconv.ParseFloat(fields[1], 64)
framework.ExpectNoError(err) framework.ExpectNoError(err)
res = append(res, containerOutput{ res = append(res, containerOutput{
timestamp: time.Unix(timestamp, 0), timestamp: timestamp,
timeSinceBoot: timeSinceBoot, containerName: fields[1],
containerName: fields[2], command: fields[2],
command: fields[3],
}) })
} }
// sort using the timeSinceBoot since it has more precision // sort using the timeSinceBoot since it has more precision
sort.Slice(res, func(i, j int) bool { sort.Slice(res, func(i, j int) bool {
return res[i].timeSinceBoot < res[j].timeSinceBoot return res[i].timestamp.Before(res[j].timestamp)
}) })
return res return res
} }

30
test/e2e_node/container_lifecycle_test.go
View File

@@ -1161,7 +1161,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 0, ExitCode: 0,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -1232,7 +1232,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 1, ExitCode: 1,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -1371,7 +1371,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 1, ExitCode: 1,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -1564,7 +1564,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 0, ExitCode: 0,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -1638,7 +1638,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 1, ExitCode: 1,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -1786,7 +1786,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 1, ExitCode: 1,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -1985,7 +1985,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 0, ExitCode: 0,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -2055,7 +2055,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 1, ExitCode: 1,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -2199,7 +2199,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
Name: restartableInit1, Name: restartableInit1,
Image: busyboxImage, Image: busyboxImage,
Command: ExecCommand(restartableInit1, execCommand{ Command: ExecCommand(restartableInit1, execCommand{
Delay: 1, Delay: 5,
ExitCode: 1, ExitCode: 1,
}), }),
RestartPolicy: &containerRestartPolicyAlways, RestartPolicy: &containerRestartPolicyAlways,
@@ -2703,7 +2703,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
ps3Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit3)) ps3Last, err := results.TimeOfLastLoop(prefixedName(PreStopPrefix, restartableInit3))
framework.ExpectNoError(err) framework.ExpectNoError(err)
const simulToleration = 0.5 const simulToleration = 500 // milliseconds
// should all end together since they loop infinitely and exceed their grace period // should all end together since they loop infinitely and exceed their grace period
gomega.Expect(ps1Last-ps2Last).To(gomega.BeNumerically("~", 0, simulToleration), gomega.Expect(ps1Last-ps2Last).To(gomega.BeNumerically("~", 0, simulToleration),
fmt.Sprintf("expected PostStart 1 & PostStart 2 to be killed at the same time, got %s", results)) fmt.Sprintf("expected PostStart 1 & PostStart 2 to be killed at the same time, got %s", results))
@@ -2713,12 +2713,12 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
fmt.Sprintf("expected PostStart 2 & PostStart 3 to be killed at the same time, got %s", results)) fmt.Sprintf("expected PostStart 2 & PostStart 3 to be killed at the same time, got %s", results))
// 30 seconds + 2 second minimum grace for the SIGKILL // 30 seconds + 2 second minimum grace for the SIGKILL
const lifetimeToleration = 1 const lifetimeToleration = 1000 // milliseconds
gomega.Expect(ps1Last-ps1).To(gomega.BeNumerically("~", 32, lifetimeToleration), gomega.Expect(ps1Last-ps1).To(gomega.BeNumerically("~", 32000, lifetimeToleration),
fmt.Sprintf("expected PostStart 1 to live for ~32 seconds, got %s", results)) fmt.Sprintf("expected PostStart 1 to live for ~32 seconds, got %s", results))
gomega.Expect(ps2Last-ps2).To(gomega.BeNumerically("~", 32, lifetimeToleration), gomega.Expect(ps2Last-ps2).To(gomega.BeNumerically("~", 32000, lifetimeToleration),
fmt.Sprintf("expected PostStart 2 to live for ~32 seconds, got %s", results)) fmt.Sprintf("expected PostStart 2 to live for ~32 seconds, got %s", results))
gomega.Expect(ps3Last-ps3).To(gomega.BeNumerically("~", 32, lifetimeToleration), gomega.Expect(ps3Last-ps3).To(gomega.BeNumerically("~", 32000, lifetimeToleration),
fmt.Sprintf("expected PostStart 3 to live for ~32 seconds, got %s", results)) fmt.Sprintf("expected PostStart 3 to live for ~32 seconds, got %s", results))
}) })
@@ -2835,7 +2835,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
ps3, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit3)) ps3, err := results.TimeOfStart(prefixedName(PreStopPrefix, restartableInit3))
framework.ExpectNoError(err) framework.ExpectNoError(err)
const toleration = 0.5 const toleration = 500 // milliseconds
gomega.Expect(ps1-ps2).To(gomega.BeNumerically("~", 0, toleration), gomega.Expect(ps1-ps2).To(gomega.BeNumerically("~", 0, toleration),
fmt.Sprintf("expected PostStart 1 & PostStart 2 to start at the same time, got %s", results)) fmt.Sprintf("expected PostStart 1 & PostStart 2 to start at the same time, got %s", results))
gomega.Expect(ps1-ps3).To(gomega.BeNumerically("~", 0, toleration), gomega.Expect(ps1-ps3).To(gomega.BeNumerically("~", 0, toleration),