Ensure that Node lease has OwnerReference set

This commit is contained in:
wojtekt 2019-11-08 17:55:12 +01:00
parent fc8f5a6410
commit 07200a0764
3 changed files with 73 additions and 12 deletions

View File

@ -108,7 +108,7 @@ func (c *controller) sync() {
lease, created := c.backoffEnsureLease() lease, created := c.backoffEnsureLease()
c.latestLease = lease c.latestLease = lease
// we don't need to update the lease if we just created it // we don't need to update the lease if we just created it
if !created { if !created && lease != nil {
if err := c.retryUpdateLease(lease); err != nil { if err := c.retryUpdateLease(lease); err != nil {
klog.Errorf("%v, will retry after %v", err, c.renewInterval) klog.Errorf("%v, will retry after %v", err, c.renewInterval)
} }
@ -144,8 +144,15 @@ func (c *controller) backoffEnsureLease() (*coordinationv1.Lease, bool) {
func (c *controller) ensureLease() (*coordinationv1.Lease, bool, error) { func (c *controller) ensureLease() (*coordinationv1.Lease, bool, error) {
lease, err := c.leaseClient.Get(c.holderIdentity, metav1.GetOptions{}) lease, err := c.leaseClient.Get(c.holderIdentity, metav1.GetOptions{})
if apierrors.IsNotFound(err) { if apierrors.IsNotFound(err) {
// lease does not exist, create it // lease does not exist, create it.
lease, err := c.leaseClient.Create(c.newLease(nil)) leaseToCreate := c.newLease(nil)
if len(leaseToCreate.OwnerReferences) == 0 {
// We want to ensure that a lease will always have OwnerReferences set.
// Thus, given that we weren't able to set it correctly, we simply
// not create it this time - we will retry in the next iteration.
return nil, false, nil
}
lease, err := c.leaseClient.Create(leaseToCreate)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }

View File

@ -308,14 +308,17 @@ func TestUpdateUsingLatestLease(t *testing.T) {
cases := []struct { cases := []struct {
desc string desc string
existingObjs []runtime.Object
latestLease *coordinationv1.Lease latestLease *coordinationv1.Lease
updateReactor func(action clienttesting.Action) (bool, runtime.Object, error) updateReactor func(action clienttesting.Action) (bool, runtime.Object, error)
getReactor func(action clienttesting.Action) (bool, runtime.Object, error) getReactor func(action clienttesting.Action) (bool, runtime.Object, error)
createReactor func(action clienttesting.Action) (bool, runtime.Object, error) createReactor func(action clienttesting.Action) (bool, runtime.Object, error)
expectLatestLease bool
expectLeaseResourceVersion string expectLeaseResourceVersion string
}{ }{
{ {
desc: "latestLease is nil and need to create", desc: "latestLease is nil and need to create",
existingObjs: []runtime.Object{node},
latestLease: nil, latestLease: nil,
updateReactor: nil, updateReactor: nil,
getReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { getReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
@ -324,30 +327,50 @@ func TestUpdateUsingLatestLease(t *testing.T) {
createReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { createReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
return true, makeLease(nodeName, "1"), nil return true, makeLease(nodeName, "1"), nil
}, },
expectLatestLease: true,
expectLeaseResourceVersion: "1", expectLeaseResourceVersion: "1",
}, },
{ {
desc: "latestLease is nil and need to update", desc: "latestLease is nil and need to create, node doesn't exist",
latestLease: nil, existingObjs: nil,
latestLease: nil,
updateReactor: nil,
getReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
return true, nil, notFoundErr
},
createReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
return true, makeLease(nodeName, "1"), nil
},
expectLatestLease: false,
expectLeaseResourceVersion: "1",
},
{
desc: "latestLease is nil and need to update",
existingObjs: []runtime.Object{node},
latestLease: nil,
updateReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { updateReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
return true, makeLease(nodeName, "2"), nil return true, makeLease(nodeName, "2"), nil
}, },
getReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { getReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
return true, makeLease(nodeName, "1"), nil return true, makeLease(nodeName, "1"), nil
}, },
expectLatestLease: true,
expectLeaseResourceVersion: "2", expectLeaseResourceVersion: "2",
}, },
{ {
desc: "latestLease exist and need to update", desc: "latestLease exist and need to update",
latestLease: makeLease(nodeName, "1"), existingObjs: []runtime.Object{node},
latestLease: makeLease(nodeName, "1"),
updateReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { updateReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
return true, makeLease(nodeName, "2"), nil return true, makeLease(nodeName, "2"), nil
}, },
expectLatestLease: true,
expectLeaseResourceVersion: "2", expectLeaseResourceVersion: "2",
}, },
{ {
desc: "update with latest lease failed", desc: "update with latest lease failed",
latestLease: makeLease(nodeName, "1"), existingObjs: []runtime.Object{node},
latestLease: makeLease(nodeName, "1"),
updateReactor: func() func(action clienttesting.Action) (bool, runtime.Object, error) { updateReactor: func() func(action clienttesting.Action) (bool, runtime.Object, error) {
i := 0 i := 0
return func(action clienttesting.Action) (bool, runtime.Object, error) { return func(action clienttesting.Action) (bool, runtime.Object, error) {
@ -366,12 +389,13 @@ func TestUpdateUsingLatestLease(t *testing.T) {
getReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { getReactor: func(action clienttesting.Action) (bool, runtime.Object, error) {
return true, makeLease(nodeName, "2"), nil return true, makeLease(nodeName, "2"), nil
}, },
expectLatestLease: true,
expectLeaseResourceVersion: "3", expectLeaseResourceVersion: "3",
}, },
} }
for _, tc := range cases { for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) { t.Run(tc.desc, func(t *testing.T) {
cl := fake.NewSimpleClientset(node) cl := fake.NewSimpleClientset(tc.existingObjs...)
if tc.updateReactor != nil { if tc.updateReactor != nil {
cl.PrependReactor("update", "leases", tc.updateReactor) cl.PrependReactor("update", "leases", tc.updateReactor)
} }
@ -392,8 +416,14 @@ func TestUpdateUsingLatestLease(t *testing.T) {
c.sync() c.sync()
if tc.expectLeaseResourceVersion != c.latestLease.ResourceVersion { if tc.expectLatestLease {
t.Fatalf("latestLease RV got %v, expected %v", c.latestLease.ResourceVersion, tc.expectLeaseResourceVersion) if tc.expectLeaseResourceVersion != c.latestLease.ResourceVersion {
t.Fatalf("latestLease RV got %v, expected %v", c.latestLease.ResourceVersion, tc.expectLeaseResourceVersion)
}
} else {
if c.latestLease != nil {
t.Fatalf("unexpected latestLease: %v", c.latestLease)
}
} }
}) })
} }

View File

@ -84,6 +84,30 @@ var _ = framework.KubeDescribe("NodeLease", func() {
time.Duration(*lease.Spec.LeaseDurationSeconds/4)*time.Second) time.Duration(*lease.Spec.LeaseDurationSeconds/4)*time.Second)
}) })
ginkgo.It("should have OwnerReferences set", func() {
leaseClient := f.ClientSet.CoordinationV1().Leases(v1.NamespaceNodeLease)
var (
err error
leaseList *coordinationv1.LeaseList
)
gomega.Eventually(func() error {
leaseList, err = leaseClient.List(metav1.ListOptions{})
if err != nil {
return err
}
return nil
}, 5*time.Minute, 5*time.Second).Should(gomega.BeNil())
// All the leases should have OwnerReferences set to their corresponding
// Node object.
for i := range leaseList.Items {
lease := &leaseList.Items[i]
ownerRefs := lease.ObjectMeta.OwnerReferences
framework.ExpectEqual(len(ownerRefs), 1)
framework.ExpectEqual(ownerRefs[0].Kind, v1.SchemeGroupVersion.WithKind("Node").Kind)
framework.ExpectEqual(ownerRefs[0].APIVersion, v1.SchemeGroupVersion.WithKind("Node").Version)
}
})
ginkgo.It("the kubelet should report node status infrequently", func() { ginkgo.It("the kubelet should report node status infrequently", func() {
ginkgo.By("wait until node is ready") ginkgo.By("wait until node is ready")
e2enode.WaitForNodeToBeReady(f.ClientSet, nodeName, 5*time.Minute) e2enode.WaitForNodeToBeReady(f.ClientSet, nodeName, 5*time.Minute)