diff --git a/process.go b/process.go
index 8d6edf4d8..4a341dc37 100644
--- a/process.go
+++ b/process.go
@@ -190,10 +190,6 @@ func (p *process) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (uint32
 	case Running, Paused, Pausing:
 		return UnknownExitStatus, errors.Wrapf(errdefs.ErrFailedPrecondition, "process must be stopped before deletion")
 	}
-	if p.io != nil {
-		p.io.Wait()
-		p.io.Close()
-	}
 	r, err := p.task.client.TaskService().DeleteProcess(ctx, &tasks.DeleteProcessRequest{
 		ContainerID: p.task.id,
 		ExecID:      p.id,
@@ -201,6 +197,10 @@ func (p *process) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (uint32
 	if err != nil {
 		return UnknownExitStatus, errdefs.FromGRPC(err)
 	}
+	if p.io != nil {
+		p.io.Wait()
+		p.io.Close()
+	}
 	return r.ExitStatus, nil
 }
 
diff --git a/windows/task.go b/windows/task.go
index 80a301194..86078f71d 100644
--- a/windows/task.go
+++ b/windows/task.go
@@ -240,6 +240,13 @@ func (t *task) DeleteProcess(ctx context.Context, id string) (*runtime.Exit, err
 		if err != nil {
 			return nil, err
 		}
+
+		// If we never started the process close the pipes
+		if p.Status() == runtime.CreatedStatus {
+			p.io.Close()
+			ea = time.Now()
+		}
+
 		t.removeProcess(id)
 		return &runtime.Exit{
 			Pid:       p.pid,