github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #75206 from liggitt/node-lease

Browse Source 
deflake nodelease test
This commit is contained in:
Kubernetes Prow Robot
2019-03-11 20:24:36 -07:00
committed by GitHub
parent bb8ccb6ce7 5b0099785e
commit c094f873a9
1 changed files with 34 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
test/e2e/common/node_lease.go
View File

@@ -101,20 +101,37 @@ var _ = framework.KubeDescribe("NodeLease", func() {
By("verify NodeStatus report period is longer than lease duration") By("verify NodeStatus report period is longer than lease duration")
// NodeStatus is reported from node to master when there is some change or // NodeStatus is reported from node to master when there is some change or
// enough time has passed. So for here, keep checking the time diff // enough time has passed. So for here, keep checking the time diff
// between 2 NodeStatus report, until it is longer than lease duration ( // between 2 NodeStatus report, until it is longer than lease duration
// the same as nodeMonitorGracePeriod). // (the same as nodeMonitorGracePeriod), or it doesn't change for at least leaseDuration
heartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, metav1.Time{}) lastHeartbeatTime := getReadyConditionHeartbeatTime(f.ClientSet, nodeName)
lastObserved := time.Now()
Eventually(func() error { Eventually(func() error {
nextHeartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, heartbeatTime) currentHeartbeatTime := getReadyConditionHeartbeatTime(f.ClientSet, nodeName)
currentObserved := time.Now()
if nextHeartbeatTime.Time.After(heartbeatTime.Time.Add(leaseDuration)) { switch {
case currentHeartbeatTime == lastHeartbeatTime:
if currentObserved.Sub(lastObserved) > 2*leaseDuration {
// heartbeat hasn't changed while watching for at least 2*leaseDuration, success!
framework.Logf("node status heartbeat is unchanged for %s, was waiting for at least %s, success!", currentObserved.Sub(lastObserved), 2*leaseDuration)
return nil return nil
} }
heartbeatTime = nextHeartbeatTime framework.Logf("node status heartbeat is unchanged for %s, waiting for %s", currentObserved.Sub(lastObserved), 2*leaseDuration)
return fmt.Errorf("node status report period is shorter than lease duration") return fmt.Errorf("node status heartbeat is unchanged for %s, waiting for %s", currentObserved.Sub(lastObserved), 2*leaseDuration)
// Enter next round immediately. case currentHeartbeatTime != lastHeartbeatTime:
}, 5*time.Minute, time.Nanosecond).Should(BeNil()) if currentHeartbeatTime.Sub(lastHeartbeatTime) > leaseDuration {
// heartbeat time changed, but the diff was greater than leaseDuration, success!
framework.Logf("node status heartbeat changed in %s, was waiting for at least %s, success!", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration)
return nil
}
lastHeartbeatTime = currentHeartbeatTime
lastObserved = currentObserved
framework.Logf("node status heartbeat changed in %s, waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration)
return fmt.Errorf("node status heartbeat changed in %s, waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration)
}
return nil
}, 5*time.Minute, time.Second).Should(BeNil())
By("verify node is still in ready status even though node status report is infrequent") By("verify node is still in ready status even though node status report is infrequent")
// This check on node status is only meaningful when this e2e test is // This check on node status is only meaningful when this e2e test is
@@ -128,22 +145,12 @@ var _ = framework.KubeDescribe("NodeLease", func() {
}) })
}) })
func getNextReadyConditionHeartbeatTime(clientSet clientset.Interface, nodeName string, prevHeartbeatTime metav1.Time) metav1.Time { func getReadyConditionHeartbeatTime(clientSet clientset.Interface, nodeName string) time.Time {
var newHeartbeatTime metav1.Time
Eventually(func() error {
node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil { Expect(err).To(BeNil())
return err
}
_, readyCondition := testutils.GetNodeCondition(&node.Status, corev1.NodeReady) _, readyCondition := testutils.GetNodeCondition(&node.Status, corev1.NodeReady)
Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue)) Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue))
newHeartbeatTime = readyCondition.LastHeartbeatTime return readyCondition.LastHeartbeatTime.Time
if prevHeartbeatTime.Before(&newHeartbeatTime) {
return nil
}
return fmt.Errorf("heartbeat has not changed yet")
}, 5*time.Minute, 5*time.Second).Should(BeNil())
return newHeartbeatTime
} }
func expectLease(lease *coordv1beta1.Lease, nodeName string) error { func expectLease(lease *coordv1beta1.Lease, nodeName string) error {