diff --git a/virtio-devices/src/balloon.rs b/virtio-devices/src/balloon.rs index 5f9119fdf..336aac832 100644 --- a/virtio-devices/src/balloon.rs +++ b/virtio-devices/src/balloon.rs @@ -620,7 +620,7 @@ impl VirtioDevice for Balloon { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -644,7 +644,7 @@ impl VirtioDevice for Balloon { let mut handler = BalloonEpollHandler { mem, queues: virtqueues, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), inflate_queue_evt, deflate_queue_evt, reporting_queue_evt, @@ -664,6 +664,8 @@ impl VirtioDevice for Balloon { Thread::VirtioBalloon, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.common.epoll_threads = Some(epoll_threads); diff --git a/virtio-devices/src/block.rs b/virtio-devices/src/block.rs index f72370443..196d4d7c2 100644 --- a/virtio-devices/src/block.rs +++ b/virtio-devices/src/block.rs @@ -1159,6 +1159,8 @@ impl VirtioDevice for Block { Thread::VirtioBlock, &mut epoll_threads, &self.exit_evt, + self.device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; } diff --git a/virtio-devices/src/console.rs b/virtio-devices/src/console.rs index 180be8dfa..c66b37532 100644 --- a/virtio-devices/src/console.rs +++ b/virtio-devices/src/console.rs @@ -719,7 +719,7 @@ impl VirtioDevice for Console { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; self.resizer @@ -741,7 +741,7 @@ impl VirtioDevice for Console { mem, input_queue, output_queue, - interrupt_cb, + interrupt_cb.clone(), self.in_buffer.clone(), Arc::clone(&self.resizer), self.endpoint.clone(), @@ -764,6 +764,8 @@ impl VirtioDevice for Console { Thread::VirtioConsole, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/iommu.rs b/virtio-devices/src/iommu.rs index b56d44b20..3d2f1cf28 100644 --- a/virtio-devices/src/iommu.rs +++ b/virtio-devices/src/iommu.rs @@ -1282,7 +1282,7 @@ impl VirtioDevice for Iommu { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -1294,7 +1294,7 @@ impl VirtioDevice for Iommu { mem, request_queue, _event_queue, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), request_queue_evt, _event_queue_evt, kill_evt, @@ -1314,6 +1314,8 @@ impl VirtioDevice for Iommu { Thread::VirtioIommu, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/mem.rs b/virtio-devices/src/mem.rs index e03252d1b..9f149deeb 100644 --- a/virtio-devices/src/mem.rs +++ b/virtio-devices/src/mem.rs @@ -953,7 +953,7 @@ impl VirtioDevice for Mem { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -967,7 +967,7 @@ impl VirtioDevice for Mem { blocks_state: Arc::clone(&self.blocks_state), config: self.config.clone(), queue, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -1000,6 +1000,8 @@ impl VirtioDevice for Mem { Thread::VirtioMem, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.common.epoll_threads = Some(epoll_threads); diff --git a/virtio-devices/src/net.rs b/virtio-devices/src/net.rs index f85192e27..56d2161ef 100644 --- a/virtio-devices/src/net.rs +++ b/virtio-devices/src/net.rs @@ -769,6 +769,8 @@ impl VirtioDevice for Net { Thread::VirtioNetCtl, &mut epoll_threads, &self.exit_evt, + self.device_status.clone(), + interrupt_cb.clone(), move || ctrl_handler.run_ctrl(&paused, paused_sync.as_ref().unwrap()), )?; self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0)); @@ -847,6 +849,8 @@ impl VirtioDevice for Net { Thread::VirtioNet, &mut epoll_threads, &self.exit_evt, + self.device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; } diff --git a/virtio-devices/src/pmem.rs b/virtio-devices/src/pmem.rs index e0ec26531..8da69ea76 100644 --- a/virtio-devices/src/pmem.rs +++ b/virtio-devices/src/pmem.rs @@ -394,7 +394,7 @@ impl VirtioDevice for Pmem { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -410,7 +410,7 @@ impl VirtioDevice for Pmem { mem, queue, disk, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -427,6 +427,8 @@ impl VirtioDevice for Pmem { Thread::VirtioPmem, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/rng.rs b/virtio-devices/src/rng.rs index b639cc192..df596813b 100644 --- a/virtio-devices/src/rng.rs +++ b/virtio-devices/src/rng.rs @@ -267,7 +267,7 @@ impl VirtioDevice for Rng { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -284,7 +284,7 @@ impl VirtioDevice for Rng { mem, queue, random_file, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -300,6 +300,8 @@ impl VirtioDevice for Rng { Thread::VirtioRng, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/thread_helper.rs b/virtio-devices/src/thread_helper.rs index 343e7c167..f89ca9f82 100644 --- a/virtio-devices/src/thread_helper.rs +++ b/virtio-devices/src/thread_helper.rs @@ -4,22 +4,27 @@ // use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use std::sync::atomic::AtomicU8; use std::thread::{self, JoinHandle}; use log::error; use seccompiler::{SeccompAction, apply_filter}; use vmm_sys_util::eventfd::EventFd; -use crate::ActivateError; use crate::epoll_helper::EpollHelperError; use crate::seccomp_filters::{Thread, get_seccomp_filter}; +use crate::{ActivateError, VirtioInterrupt, mark_device_needs_reset}; +#[allow(clippy::too_many_arguments)] pub(crate) fn spawn_virtio_thread( name: &str, seccomp_action: &SeccompAction, thread_type: Thread, epoll_threads: &mut Vec>, exit_evt: &EventFd, + device_status: Arc, + interrupt_cb: Arc, f: F, ) -> Result<(), ActivateError> where @@ -49,12 +54,14 @@ where error!("{thread_name} thread panicked"); thread_exit_evt.write(1).ok(); } - Ok(r) => { - if let Err(e) = r { - error!("Error running worker: {e:?}"); - thread_exit_evt.write(1).ok(); - } + Ok(Err(e)) => { + mark_device_needs_reset( + &device_status, + interrupt_cb.as_ref(), + format_args!("{thread_name}: worker exited with error: {e:?}"), + ); } + Ok(Ok(())) => {} } }) .map(|thread| epoll_threads.push(thread)) diff --git a/virtio-devices/src/vhost_user/blk.rs b/virtio-devices/src/vhost_user/blk.rs index cddffc5d6..d2ae3ca5b 100644 --- a/virtio-devices/src/vhost_user/blk.rs +++ b/virtio-devices/src/vhost_user/blk.rs @@ -266,7 +266,7 @@ impl VirtioDevice for Blk { mem, interrupt_cb, queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -282,7 +282,7 @@ impl VirtioDevice for Blk { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), self.vu_common.virtio_common.acked_features, backend_req_handler, kill_evt, @@ -300,6 +300,8 @@ impl VirtioDevice for Blk { Thread::VirtioVhostBlock, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vhost_user/fs.rs b/virtio-devices/src/vhost_user/fs.rs index 37c7c420e..660cbb480 100644 --- a/virtio-devices/src/vhost_user/fs.rs +++ b/virtio-devices/src/vhost_user/fs.rs @@ -245,7 +245,7 @@ impl VirtioDevice for Fs { mem, interrupt_cb, queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -260,7 +260,7 @@ impl VirtioDevice for Fs { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), self.vu_common.virtio_common.acked_features, backend_req_handler, kill_evt, @@ -277,6 +277,8 @@ impl VirtioDevice for Fs { Thread::VirtioVhostFs, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vhost_user/generic_vhost_user.rs b/virtio-devices/src/vhost_user/generic_vhost_user.rs index 8397744ab..3024b1976 100644 --- a/virtio-devices/src/vhost_user/generic_vhost_user.rs +++ b/virtio-devices/src/vhost_user/generic_vhost_user.rs @@ -283,7 +283,7 @@ impl VirtioDevice for GenericVhostUser { mem, interrupt_cb, queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -313,7 +313,7 @@ impl VirtioDevice for GenericVhostUser { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), self.vu_common.virtio_common.acked_features, backend_req_handler, kill_evt, @@ -330,6 +330,8 @@ impl VirtioDevice for GenericVhostUser { Thread::VirtioGenericVhostUser, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vhost_user/net.rs b/virtio-devices/src/vhost_user/net.rs index d77c80a51..e6e01f8f0 100644 --- a/virtio-devices/src/vhost_user/net.rs +++ b/virtio-devices/src/vhost_user/net.rs @@ -270,7 +270,7 @@ impl VirtioDevice for Net { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -321,6 +321,8 @@ impl VirtioDevice for Net { Thread::VirtioVhostNetCtl, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || ctrl_handler.run_ctrl(&paused, paused_sync.as_ref().unwrap()), )?; self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0)); @@ -340,7 +342,7 @@ impl VirtioDevice for Net { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), backend_acked_features, backend_req_handler, kill_evt, @@ -357,6 +359,8 @@ impl VirtioDevice for Net { Thread::VirtioVhostNet, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vsock/device.rs b/virtio-devices/src/vsock/device.rs index 8247e0398..ef79e7d78 100644 --- a/virtio-devices/src/vsock/device.rs +++ b/virtio-devices/src/vsock/device.rs @@ -480,7 +480,7 @@ where mem, interrupt_cb, queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -498,7 +498,7 @@ where queue_evts, kill_evt, pause_evt, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), backend: self.backend.clone(), access_platform: self.common.access_platform(), }; @@ -513,6 +513,8 @@ where Thread::VirtioVsock, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/watchdog.rs b/virtio-devices/src/watchdog.rs index e362532e6..38fa5b6de 100644 --- a/virtio-devices/src/watchdog.rs +++ b/virtio-devices/src/watchdog.rs @@ -331,7 +331,7 @@ impl VirtioDevice for Watchdog { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -351,7 +351,7 @@ impl VirtioDevice for Watchdog { let mut handler = WatchdogEpollHandler { mem, queue, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -370,6 +370,8 @@ impl VirtioDevice for Watchdog { Thread::VirtioWatchdog, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?;