From 101d4b78eb7fc0ef2be6dea1cfe2e5cf277df622 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 23 Jul 2019 19:52:42 +0000 Subject: [PATCH] Add timeout to task state calls Fixes #3440 This also returns the task that times out or has an error on the state call in an UNKNOWN status so that the user can manually kill and remove the task. Signed-off-by: Michael Crosby --- services/tasks/local.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/services/tasks/local.go b/services/tasks/local.go index 63aff1b8a..163ed0186 100644 --- a/services/tasks/local.go +++ b/services/tasks/local.go @@ -266,12 +266,18 @@ func (l *local) DeleteProcess(ctx context.Context, r *api.DeleteProcessRequest, }, nil } -func processFromContainerd(ctx context.Context, p runtime.Process) (*task.Process, error) { +func getProcessState(ctx context.Context, p runtime.Process) (*task.Process, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + state, err := p.State(ctx) if err != nil { - return nil, err + if errdefs.IsNotFound(err) { + return nil, err + } + log.G(ctx).WithError(err).Errorf("get state for %s", p.ID()) } - var status task.Status + status := task.StatusUnknown switch state.Status { case runtime.CreatedStatus: status = task.StatusCreated @@ -310,7 +316,7 @@ func (l *local) Get(ctx context.Context, r *api.GetRequest, _ ...grpc.CallOption return nil, errdefs.ToGRPC(err) } } - t, err := processFromContainerd(ctx, p) + t, err := getProcessState(ctx, p) if err != nil { return nil, errdefs.ToGRPC(err) } @@ -333,7 +339,7 @@ func (l *local) List(ctx context.Context, r *api.ListTasksRequest, _ ...grpc.Cal func addTasks(ctx context.Context, r *api.ListTasksResponse, tasks []runtime.Task) { for _, t := range tasks { - tt, err := processFromContainerd(ctx, t) + tt, err := getProcessState(ctx, t) if err != nil { if !errdefs.IsNotFound(err) { // handle race with deletion log.G(ctx).WithError(err).WithField("id", t.ID()).Error("converting task to protobuf")