Alter graceful deletion to not use TTL

Avoid TTL by deleting pods immediately when they aren't
scheduled, and letting the Kubelet delete them otherwise.

Ensure the Kubelet uses pod.Spec.TerminationGracePeriodSeconds
when no pod.DeletionGracePeriodSeconds is available.

pkg/registry/generic/etcd/etcd.go

@@ -341,6 +341,11 @@ func (e *Etcd) Get(ctx api.Context, name string) (runtime.Object, error) {
 	return obj, nil
 }
 
+var (
+	errAlreadyDeleting = fmt.Errorf("abort delete")
+	errDeleteNow       = fmt.Errorf("delete now")
+)
+
 // Delete removes the item from etcd.
 func (e *Etcd) Delete(ctx api.Context, name string, options *api.DeleteOptions) (runtime.Object, error) {
 	key, err := e.KeyFunc(ctx, name)
@@ -367,13 +372,41 @@ func (e *Etcd) Delete(ctx api.Context, name string, options *api.DeleteOptions)
 	if pendingGraceful {
 		return e.finalizeDelete(obj, false)
 	}
-	if graceful && *options.GracePeriodSeconds != 0 {
+	if graceful {
 		trace.Step("Graceful deletion")
 		out := e.NewFunc()
-		if err := e.Storage.Set(key, obj, out, uint64(*options.GracePeriodSeconds)); err != nil {
+		lastGraceful := int64(0)
+		err := e.Storage.GuaranteedUpdate(
+			key, out, false,
+			storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
+				graceful, pendingGraceful, err := rest.BeforeDelete(e.DeleteStrategy, ctx, existing, options)
+				if err != nil {
+					return nil, err
+				}
+				if pendingGraceful {
+					return nil, errAlreadyDeleting
+				}
+				if !graceful {
+					return nil, errDeleteNow
+				}
+				lastGraceful = *options.GracePeriodSeconds
+				return existing, nil
+			}),
+		)
+		switch err {
+		case nil:
+			if lastGraceful > 0 {
+				return out, nil
+			}
+			// fall through and delete immediately
+		case errDeleteNow:
+			// we've updated the object to have a zero grace period, or it's already at 0, so
+			// we should fall through and truly delete the object.
+		case errAlreadyDeleting:
+			return e.finalizeDelete(obj, true)
+		default:
 			return nil, etcderr.InterpretUpdateError(err, e.EndpointName, name)
 		}
-		return e.finalizeDelete(out, true)
 	}
 
 	// delete immediately, or no graceful deletion supported

pkg/registry/pod/etcd/etcd_test.go

@@ -121,8 +121,10 @@ func TestDelete(t *testing.T) {
 	key = etcdtest.AddPrefix(key)
 	test := resttest.New(t, storage, fakeEtcdClient.SetError)
 
+	expectedNode := "some-node"
 	createFn := func() runtime.Object {
 		pod := validChangedPod()
+		pod.Spec.NodeName = expectedNode
 		fakeEtcdClient.Data[key] = tools.EtcdResponseWithError{
 			R: &etcd.Response{
 				Node: &etcd.Node{
@@ -137,9 +139,18 @@ func TestDelete(t *testing.T) {
 		if fakeEtcdClient.Data[key].R.Node == nil {
 			return false
 		}
-		return fakeEtcdClient.Data[key].R.Node.TTL != 0
+		obj, err := latest.Codec.Decode([]byte(fakeEtcdClient.Data[key].R.Node.Value))
+		if err != nil {
+			return false
+		}
+		pod := obj.(*api.Pod)
+		t.Logf("found object %#v", pod.ObjectMeta)
+		return pod.DeletionTimestamp != nil && pod.DeletionGracePeriodSeconds != nil && *pod.DeletionGracePeriodSeconds != 0
 	}
 	test.TestDeleteGraceful(createFn, 30, gracefulSetFn)
+
+	expectedNode = ""
+	test.TestDelete(createFn, gracefulSetFn)
 }
 
 func expectPod(t *testing.T, out runtime.Object) (*api.Pod, bool) {

pkg/registry/pod/rest.go

@@ -92,22 +92,38 @@ func (podStrategy) CheckGracefulDelete(obj runtime.Object, options *api.DeleteOp
 	if options == nil {
 		return false
 	}
+	pod := obj.(*api.Pod)
 	period := int64(0)
 	// user has specified a value
 	if options.GracePeriodSeconds != nil {
 		period = *options.GracePeriodSeconds
 	} else {
 		// use the default value if set, or deletes the pod immediately (0)
-		pod := obj.(*api.Pod)
 		if pod.Spec.TerminationGracePeriodSeconds != nil {
 			period = *pod.Spec.TerminationGracePeriodSeconds
 		}
 	}
+	// if the pod is not scheduled, delete immediately
+	if len(pod.Spec.NodeName) == 0 {
+		period = 0
+	}
 	// ensure the options and the pod are in sync
 	options.GracePeriodSeconds = &period
 	return true
 }
 
+type podStrategyWithoutGraceful struct {
+	podStrategy
+}
+
+// CheckGracefulDelete prohibits graceful deletion.
+func (podStrategyWithoutGraceful) CheckGracefulDelete(obj runtime.Object, options *api.DeleteOptions) bool {
+	return false
+}
+
+// StrategyWithoutGraceful implements the legacy instant delele behavior.
+var StrategyWithoutGraceful = podStrategyWithoutGraceful{Strategy}
+
 type podStatusStrategy struct {
 	podStrategy
 }