diff --git a/container_test.go b/container_test.go
index ee09d81ac..ab4ecbba4 100644
--- a/container_test.go
+++ b/container_test.go
@@ -805,7 +805,7 @@ func TestContainerNoBinaryExists(t *testing.T) {
 		}
 	}
 
-	spec, err := generateSpec(withImageConfig(ctx, image), withProcessArgs("nothing"))
+	spec, err := generateSpec(withImageConfig(ctx, image), WithProcessArgs("nothing"))
 	if err != nil {
 		t.Error(err)
 		return
@@ -821,11 +821,12 @@ func TestContainerNoBinaryExists(t *testing.T) {
 	switch runtime.GOOS {
 	case "windows":
 		if err != nil {
-			t.Errorf("failed to create task %v", err)
+			t.Fatalf("failed to create task %v", err)
 		}
-		if err := task.Start(ctx); err != nil {
+		defer task.Delete(ctx)
+		if err := task.Start(ctx); err == nil {
+			task.Kill(ctx, syscall.SIGKILL)
 			t.Error("task.Start() should return an error when binary does not exist")
-			task.Delete(ctx)
 		}
 	default:
 		if err == nil {
diff --git a/task.go b/task.go
index 87c4defc3..2d49fad27 100644
--- a/task.go
+++ b/task.go
@@ -17,6 +17,7 @@ import (
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/rootfs"
 	"github.com/containerd/containerd/runtime"
 	"github.com/containerd/containerd/typeurl"
@@ -117,7 +118,8 @@ type Task interface {
 var _ = (Task)(&task{})
 
 type task struct {
-	client *Client
+	client    *Client
+	container Container
 
 	io  *IO
 	id  string
@@ -250,6 +252,12 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (uint32, e
 	}
 	switch status.Status {
 	case Stopped, Unknown, "":
+	case Created:
+		if t.client.runtime == fmt.Sprintf("%s.%s", plugin.RuntimePlugin, "windows") {
+			// On windows Created is akin to Stopped
+			break
+		}
+		fallthrough
 	default:
 		return UnknownExitStatus, errors.Wrapf(errdefs.ErrFailedPrecondition, "task must be stopped before deletion: %s", status.Status)
 	}
diff --git a/windows/task.go b/windows/task.go
index f8caa67cd..76c3c1a61 100644
--- a/windows/task.go
+++ b/windows/task.go
@@ -114,6 +114,7 @@ func (t *task) Start(ctx context.Context) error {
 		return err
 	}
 	if err := p.Start(ctx); err != nil {
+		t.removeProcess(t.id)
 		return err
 	}
 	t.publisher.Publish(ctx,