fix shim reaper wait command execute blocked

wait no timeout will lead to event publish
process hang in some special scenarios.

Signed-off-by: botieking98 <botieking@gmail.com>
This commit is contained in:
botieking98 2021-10-27 14:58:31 +08:00
parent aa65faebd7
commit 3e51312a61
2 changed files with 24 additions and 1 deletions

View File

@ -308,7 +308,7 @@ func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event
if err != nil { if err != nil {
return err return err
} }
status, err := reaper.Default.Wait(cmd, c) status, err := reaper.Default.WaitTimeout(cmd, c, 30*time.Second)
if err != nil { if err != nil {
return errors.Wrapf(err, "failed to publish event: %s", b.String()) return errors.Wrapf(err, "failed to publish event: %s", b.String())
} }

View File

@ -21,6 +21,7 @@ package reaper
import ( import (
"sync" "sync"
"syscall"
"time" "time"
runc "github.com/containerd/go-runc" runc "github.com/containerd/go-runc"
@ -116,6 +117,28 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) {
return -1, ErrNoSuchProcess return -1, ErrNoSuchProcess
} }
// WaitTimeout is used to skip the blocked command and kill the left process.
func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, timeout time.Duration) (int, error) {
sch := make(chan int)
ech := make(chan error)
go func() {
status, err := m.Wait(c, ec)
sch <- status
if err != nil {
ech <- err
}
}()
select {
case <-time.After(timeout):
syscall.Kill(c.Process.Pid, syscall.SIGKILL)
return 0, errors.Errorf("timeout %ds for cmd(pid=%d): %s, %s", timeout/time.Second, c.Process.Pid, c.Path, c.Args)
case status := <-sch:
return status, nil
case err := <-ech:
return -1, err
}
}
// Subscribe to process exit changes // Subscribe to process exit changes
func (m *Monitor) Subscribe() chan runc.Exit { func (m *Monitor) Subscribe() chan runc.Exit {
c := make(chan runc.Exit, bufferSize) c := make(chan runc.Exit, bufferSize)