From 3402bc0762cdee705b880a996b765d0c1377013b Mon Sep 17 00:00:00 2001 From: Anatol Belski Date: Wed, 6 May 2026 15:20:48 +0200 Subject: [PATCH] virtio-devices: NEEDS_RESET on worker thread Err Worker threads spawned through spawn_virtio_thread previously wrote to exit_evt on any clean Err return, taking the whole VMM down on a single failed device worker. A guest induced fault in any virtio device thus propagated into a host wide failure. Route the Err return through the shared mark_device_needs_reset helper instead. The helper sets the DEVICE_NEEDS_RESET bit on device_status and triggers a config change interrupt, so the device goes idle and the guest is informed. The thread exits cleanly without killing the rest of the VMM. The panic and the seccomp filter apply paths keep writing to exit_evt. A panicked worker may have left poisoned locks or partially mutated state, so a hard exit remains the right policy there. spawn_virtio_thread now takes the device_status and the interrupt callback. Every native virtio and vhost-user call site is updated to pass them in. Signed-off-by: Anatol Belski --- virtio-devices/src/balloon.rs | 6 ++++-- virtio-devices/src/block.rs | 2 ++ virtio-devices/src/console.rs | 6 ++++-- virtio-devices/src/iommu.rs | 6 ++++-- virtio-devices/src/mem.rs | 6 ++++-- virtio-devices/src/net.rs | 4 ++++ virtio-devices/src/pmem.rs | 6 ++++-- virtio-devices/src/rng.rs | 6 ++++-- virtio-devices/src/thread_helper.rs | 19 +++++++++++++------ virtio-devices/src/vhost_user/blk.rs | 6 ++++-- virtio-devices/src/vhost_user/fs.rs | 6 ++++-- .../src/vhost_user/generic_vhost_user.rs | 6 ++++-- virtio-devices/src/vhost_user/net.rs | 8 ++++++-- virtio-devices/src/vsock/device.rs | 6 ++++-- virtio-devices/src/watchdog.rs | 6 ++++-- 15 files changed, 69 insertions(+), 30 deletions(-) diff --git a/virtio-devices/src/balloon.rs b/virtio-devices/src/balloon.rs index 5f9119fdf..336aac832 100644 --- a/virtio-devices/src/balloon.rs +++ b/virtio-devices/src/balloon.rs @@ -620,7 +620,7 @@ impl VirtioDevice for Balloon { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -644,7 +644,7 @@ impl VirtioDevice for Balloon { let mut handler = BalloonEpollHandler { mem, queues: virtqueues, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), inflate_queue_evt, deflate_queue_evt, reporting_queue_evt, @@ -664,6 +664,8 @@ impl VirtioDevice for Balloon { Thread::VirtioBalloon, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.common.epoll_threads = Some(epoll_threads); diff --git a/virtio-devices/src/block.rs b/virtio-devices/src/block.rs index f72370443..196d4d7c2 100644 --- a/virtio-devices/src/block.rs +++ b/virtio-devices/src/block.rs @@ -1159,6 +1159,8 @@ impl VirtioDevice for Block { Thread::VirtioBlock, &mut epoll_threads, &self.exit_evt, + self.device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; } diff --git a/virtio-devices/src/console.rs b/virtio-devices/src/console.rs index 180be8dfa..c66b37532 100644 --- a/virtio-devices/src/console.rs +++ b/virtio-devices/src/console.rs @@ -719,7 +719,7 @@ impl VirtioDevice for Console { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; self.resizer @@ -741,7 +741,7 @@ impl VirtioDevice for Console { mem, input_queue, output_queue, - interrupt_cb, + interrupt_cb.clone(), self.in_buffer.clone(), Arc::clone(&self.resizer), self.endpoint.clone(), @@ -764,6 +764,8 @@ impl VirtioDevice for Console { Thread::VirtioConsole, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/iommu.rs b/virtio-devices/src/iommu.rs index b56d44b20..3d2f1cf28 100644 --- a/virtio-devices/src/iommu.rs +++ b/virtio-devices/src/iommu.rs @@ -1282,7 +1282,7 @@ impl VirtioDevice for Iommu { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -1294,7 +1294,7 @@ impl VirtioDevice for Iommu { mem, request_queue, _event_queue, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), request_queue_evt, _event_queue_evt, kill_evt, @@ -1314,6 +1314,8 @@ impl VirtioDevice for Iommu { Thread::VirtioIommu, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/mem.rs b/virtio-devices/src/mem.rs index e03252d1b..9f149deeb 100644 --- a/virtio-devices/src/mem.rs +++ b/virtio-devices/src/mem.rs @@ -953,7 +953,7 @@ impl VirtioDevice for Mem { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -967,7 +967,7 @@ impl VirtioDevice for Mem { blocks_state: Arc::clone(&self.blocks_state), config: self.config.clone(), queue, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -1000,6 +1000,8 @@ impl VirtioDevice for Mem { Thread::VirtioMem, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.common.epoll_threads = Some(epoll_threads); diff --git a/virtio-devices/src/net.rs b/virtio-devices/src/net.rs index f85192e27..56d2161ef 100644 --- a/virtio-devices/src/net.rs +++ b/virtio-devices/src/net.rs @@ -769,6 +769,8 @@ impl VirtioDevice for Net { Thread::VirtioNetCtl, &mut epoll_threads, &self.exit_evt, + self.device_status.clone(), + interrupt_cb.clone(), move || ctrl_handler.run_ctrl(&paused, paused_sync.as_ref().unwrap()), )?; self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0)); @@ -847,6 +849,8 @@ impl VirtioDevice for Net { Thread::VirtioNet, &mut epoll_threads, &self.exit_evt, + self.device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; } diff --git a/virtio-devices/src/pmem.rs b/virtio-devices/src/pmem.rs index e0ec26531..8da69ea76 100644 --- a/virtio-devices/src/pmem.rs +++ b/virtio-devices/src/pmem.rs @@ -394,7 +394,7 @@ impl VirtioDevice for Pmem { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -410,7 +410,7 @@ impl VirtioDevice for Pmem { mem, queue, disk, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -427,6 +427,8 @@ impl VirtioDevice for Pmem { Thread::VirtioPmem, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/rng.rs b/virtio-devices/src/rng.rs index b639cc192..df596813b 100644 --- a/virtio-devices/src/rng.rs +++ b/virtio-devices/src/rng.rs @@ -267,7 +267,7 @@ impl VirtioDevice for Rng { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -284,7 +284,7 @@ impl VirtioDevice for Rng { mem, queue, random_file, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -300,6 +300,8 @@ impl VirtioDevice for Rng { Thread::VirtioRng, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/thread_helper.rs b/virtio-devices/src/thread_helper.rs index 343e7c167..f89ca9f82 100644 --- a/virtio-devices/src/thread_helper.rs +++ b/virtio-devices/src/thread_helper.rs @@ -4,22 +4,27 @@ // use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use std::sync::atomic::AtomicU8; use std::thread::{self, JoinHandle}; use log::error; use seccompiler::{SeccompAction, apply_filter}; use vmm_sys_util::eventfd::EventFd; -use crate::ActivateError; use crate::epoll_helper::EpollHelperError; use crate::seccomp_filters::{Thread, get_seccomp_filter}; +use crate::{ActivateError, VirtioInterrupt, mark_device_needs_reset}; +#[allow(clippy::too_many_arguments)] pub(crate) fn spawn_virtio_thread( name: &str, seccomp_action: &SeccompAction, thread_type: Thread, epoll_threads: &mut Vec>, exit_evt: &EventFd, + device_status: Arc, + interrupt_cb: Arc, f: F, ) -> Result<(), ActivateError> where @@ -49,12 +54,14 @@ where error!("{thread_name} thread panicked"); thread_exit_evt.write(1).ok(); } - Ok(r) => { - if let Err(e) = r { - error!("Error running worker: {e:?}"); - thread_exit_evt.write(1).ok(); - } + Ok(Err(e)) => { + mark_device_needs_reset( + &device_status, + interrupt_cb.as_ref(), + format_args!("{thread_name}: worker exited with error: {e:?}"), + ); } + Ok(Ok(())) => {} } }) .map(|thread| epoll_threads.push(thread)) diff --git a/virtio-devices/src/vhost_user/blk.rs b/virtio-devices/src/vhost_user/blk.rs index cddffc5d6..d2ae3ca5b 100644 --- a/virtio-devices/src/vhost_user/blk.rs +++ b/virtio-devices/src/vhost_user/blk.rs @@ -266,7 +266,7 @@ impl VirtioDevice for Blk { mem, interrupt_cb, queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -282,7 +282,7 @@ impl VirtioDevice for Blk { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), self.vu_common.virtio_common.acked_features, backend_req_handler, kill_evt, @@ -300,6 +300,8 @@ impl VirtioDevice for Blk { Thread::VirtioVhostBlock, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vhost_user/fs.rs b/virtio-devices/src/vhost_user/fs.rs index 37c7c420e..660cbb480 100644 --- a/virtio-devices/src/vhost_user/fs.rs +++ b/virtio-devices/src/vhost_user/fs.rs @@ -245,7 +245,7 @@ impl VirtioDevice for Fs { mem, interrupt_cb, queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -260,7 +260,7 @@ impl VirtioDevice for Fs { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), self.vu_common.virtio_common.acked_features, backend_req_handler, kill_evt, @@ -277,6 +277,8 @@ impl VirtioDevice for Fs { Thread::VirtioVhostFs, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vhost_user/generic_vhost_user.rs b/virtio-devices/src/vhost_user/generic_vhost_user.rs index 8397744ab..3024b1976 100644 --- a/virtio-devices/src/vhost_user/generic_vhost_user.rs +++ b/virtio-devices/src/vhost_user/generic_vhost_user.rs @@ -283,7 +283,7 @@ impl VirtioDevice for GenericVhostUser { mem, interrupt_cb, queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -313,7 +313,7 @@ impl VirtioDevice for GenericVhostUser { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), self.vu_common.virtio_common.acked_features, backend_req_handler, kill_evt, @@ -330,6 +330,8 @@ impl VirtioDevice for GenericVhostUser { Thread::VirtioGenericVhostUser, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vhost_user/net.rs b/virtio-devices/src/vhost_user/net.rs index d77c80a51..e6e01f8f0 100644 --- a/virtio-devices/src/vhost_user/net.rs +++ b/virtio-devices/src/vhost_user/net.rs @@ -270,7 +270,7 @@ impl VirtioDevice for Net { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.vu_common .virtio_common @@ -321,6 +321,8 @@ impl VirtioDevice for Net { Thread::VirtioVhostNetCtl, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || ctrl_handler.run_ctrl(&paused, paused_sync.as_ref().unwrap()), )?; self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0)); @@ -340,7 +342,7 @@ impl VirtioDevice for Net { let mut handler = self.vu_common.activate( mem, &queues, - interrupt_cb, + interrupt_cb.clone(), backend_acked_features, backend_req_handler, kill_evt, @@ -357,6 +359,8 @@ impl VirtioDevice for Net { Thread::VirtioVhostNet, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; self.vu_common.epoll_thread = Some(epoll_threads.remove(0)); diff --git a/virtio-devices/src/vsock/device.rs b/virtio-devices/src/vsock/device.rs index 8247e0398..ef79e7d78 100644 --- a/virtio-devices/src/vsock/device.rs +++ b/virtio-devices/src/vsock/device.rs @@ -480,7 +480,7 @@ where mem, interrupt_cb, queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -498,7 +498,7 @@ where queue_evts, kill_evt, pause_evt, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), backend: self.backend.clone(), access_platform: self.common.access_platform(), }; @@ -513,6 +513,8 @@ where Thread::VirtioVsock, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?; diff --git a/virtio-devices/src/watchdog.rs b/virtio-devices/src/watchdog.rs index e362532e6..38fa5b6de 100644 --- a/virtio-devices/src/watchdog.rs +++ b/virtio-devices/src/watchdog.rs @@ -331,7 +331,7 @@ impl VirtioDevice for Watchdog { mem, interrupt_cb, mut queues, - .. + device_status, } = context; self.common.activate(&queues, interrupt_cb.clone())?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); @@ -351,7 +351,7 @@ impl VirtioDevice for Watchdog { let mut handler = WatchdogEpollHandler { mem, queue, - interrupt_cb, + interrupt_cb: interrupt_cb.clone(), queue_evt, kill_evt, pause_evt, @@ -370,6 +370,8 @@ impl VirtioDevice for Watchdog { Thread::VirtioWatchdog, &mut epoll_threads, &self.exit_evt, + device_status.clone(), + interrupt_cb.clone(), move || handler.run(&paused, paused_sync.as_ref().unwrap()), )?;