virtio-devices: NEEDS_RESET on worker thread Err

Worker threads spawned through spawn_virtio_thread previously wrote to
exit_evt on any clean Err return, taking the whole VMM down on a single
failed device worker. A guest induced fault in any virtio device thus
propagated into a host wide failure.

Route the Err return through the shared mark_device_needs_reset helper
instead. The helper sets the DEVICE_NEEDS_RESET bit on device_status
and triggers a config change interrupt, so the device goes idle and
the guest is informed. The thread exits cleanly without killing the
rest of the VMM.

The panic and the seccomp filter apply paths keep writing to exit_evt.
A panicked worker may have left poisoned locks or partially mutated
state, so a hard exit remains the right policy there.

spawn_virtio_thread now takes the device_status and the interrupt
callback. Every native virtio and vhost-user call site is updated to
pass them in.

Signed-off-by: Anatol Belski <anbelski@linux.microsoft.com>

virtio-devices/src/balloon.rs

@@ -620,7 +620,7 @@ impl VirtioDevice for Balloon {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
@@ -644,7 +644,7 @@ impl VirtioDevice for Balloon {
         let mut handler = BalloonEpollHandler {
             mem,
             queues: virtqueues,
-            interrupt_cb,
+            interrupt_cb: interrupt_cb.clone(),
             inflate_queue_evt,
             deflate_queue_evt,
             reporting_queue_evt,
@@ -664,6 +664,8 @@ impl VirtioDevice for Balloon {
             Thread::VirtioBalloon,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
         self.common.epoll_threads = Some(epoll_threads);

virtio-devices/src/block.rs

@@ -1159,6 +1159,8 @@ impl VirtioDevice for Block {
                 Thread::VirtioBlock,
                 &mut epoll_threads,
                 &self.exit_evt,
+                self.device_status.clone(),
+                interrupt_cb.clone(),
                 move || handler.run(&paused, paused_sync.as_ref().unwrap()),
             )?;
         }

virtio-devices/src/console.rs

@@ -719,7 +719,7 @@ impl VirtioDevice for Console {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         self.resizer
@@ -741,7 +741,7 @@ impl VirtioDevice for Console {
             mem,
             input_queue,
             output_queue,
-            interrupt_cb,
+            interrupt_cb.clone(),
             self.in_buffer.clone(),
             Arc::clone(&self.resizer),
             self.endpoint.clone(),
@@ -764,6 +764,8 @@ impl VirtioDevice for Console {
             Thread::VirtioConsole,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
 

virtio-devices/src/iommu.rs

@@ -1282,7 +1282,7 @@ impl VirtioDevice for Iommu {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
@@ -1294,7 +1294,7 @@ impl VirtioDevice for Iommu {
             mem,
             request_queue,
             _event_queue,
-            interrupt_cb,
+            interrupt_cb: interrupt_cb.clone(),
             request_queue_evt,
             _event_queue_evt,
             kill_evt,
@@ -1314,6 +1314,8 @@ impl VirtioDevice for Iommu {
             Thread::VirtioIommu,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
 

virtio-devices/src/mem.rs

@@ -953,7 +953,7 @@ impl VirtioDevice for Mem {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
@@ -967,7 +967,7 @@ impl VirtioDevice for Mem {
             blocks_state: Arc::clone(&self.blocks_state),
             config: self.config.clone(),
             queue,
-            interrupt_cb,
+            interrupt_cb: interrupt_cb.clone(),
             queue_evt,
             kill_evt,
             pause_evt,
@@ -1000,6 +1000,8 @@ impl VirtioDevice for Mem {
             Thread::VirtioMem,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
         self.common.epoll_threads = Some(epoll_threads);

virtio-devices/src/net.rs

@@ -769,6 +769,8 @@ impl VirtioDevice for Net {
                 Thread::VirtioNetCtl,
                 &mut epoll_threads,
                 &self.exit_evt,
+                self.device_status.clone(),
+                interrupt_cb.clone(),
                 move || ctrl_handler.run_ctrl(&paused, paused_sync.as_ref().unwrap()),
             )?;
             self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0));
@@ -847,6 +849,8 @@ impl VirtioDevice for Net {
                 Thread::VirtioNet,
                 &mut epoll_threads,
                 &self.exit_evt,
+                self.device_status.clone(),
+                interrupt_cb.clone(),
                 move || handler.run(&paused, paused_sync.as_ref().unwrap()),
             )?;
         }

virtio-devices/src/pmem.rs

@@ -394,7 +394,7 @@ impl VirtioDevice for Pmem {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
@@ -410,7 +410,7 @@ impl VirtioDevice for Pmem {
                 mem,
                 queue,
                 disk,
-                interrupt_cb,
+                interrupt_cb: interrupt_cb.clone(),
                 queue_evt,
                 kill_evt,
                 pause_evt,
@@ -427,6 +427,8 @@ impl VirtioDevice for Pmem {
                 Thread::VirtioPmem,
                 &mut epoll_threads,
                 &self.exit_evt,
+                device_status.clone(),
+                interrupt_cb.clone(),
                 move || handler.run(&paused, paused_sync.as_ref().unwrap()),
             )?;
 

virtio-devices/src/rng.rs

@@ -267,7 +267,7 @@ impl VirtioDevice for Rng {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
@@ -284,7 +284,7 @@ impl VirtioDevice for Rng {
                 mem,
                 queue,
                 random_file,
-                interrupt_cb,
+                interrupt_cb: interrupt_cb.clone(),
                 queue_evt,
                 kill_evt,
                 pause_evt,
@@ -300,6 +300,8 @@ impl VirtioDevice for Rng {
                 Thread::VirtioRng,
                 &mut epoll_threads,
                 &self.exit_evt,
+                device_status.clone(),
+                interrupt_cb.clone(),
                 move || handler.run(&paused, paused_sync.as_ref().unwrap()),
             )?;
 

virtio-devices/src/thread_helper.rs

@@ -4,22 +4,27 @@
 //
 
 use std::panic::AssertUnwindSafe;
+use std::sync::Arc;
+use std::sync::atomic::AtomicU8;
 use std::thread::{self, JoinHandle};
 
 use log::error;
 use seccompiler::{SeccompAction, apply_filter};
 use vmm_sys_util::eventfd::EventFd;
 
-use crate::ActivateError;
 use crate::epoll_helper::EpollHelperError;
 use crate::seccomp_filters::{Thread, get_seccomp_filter};
+use crate::{ActivateError, VirtioInterrupt, mark_device_needs_reset};
 
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn spawn_virtio_thread<F>(
     name: &str,
     seccomp_action: &SeccompAction,
     thread_type: Thread,
     epoll_threads: &mut Vec<JoinHandle<()>>,
     exit_evt: &EventFd,
+    device_status: Arc<AtomicU8>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
     f: F,
 ) -> Result<(), ActivateError>
 where
@@ -49,12 +54,14 @@ where
                     error!("{thread_name} thread panicked");
                     thread_exit_evt.write(1).ok();
                 }
-                Ok(r) => {
-                    if let Err(e) = r {
-                        error!("Error running worker: {e:?}");
-                        thread_exit_evt.write(1).ok();
-                    }
+                Ok(Err(e)) => {
+                    mark_device_needs_reset(
+                        &device_status,
+                        interrupt_cb.as_ref(),
+                        format_args!("{thread_name}: worker exited with error: {e:?}"),
+                    );
                 }
+                Ok(Ok(())) => {}
             }
         })
         .map(|thread| epoll_threads.push(thread))

virtio-devices/src/vhost_user/blk.rs

@@ -266,7 +266,7 @@ impl VirtioDevice for Blk {
             mem,
             interrupt_cb,
             queues,
-            ..
+            device_status,
         } = context;
         self.vu_common
             .virtio_common
@@ -282,7 +282,7 @@ impl VirtioDevice for Blk {
         let mut handler = self.vu_common.activate(
             mem,
             &queues,
-            interrupt_cb,
+            interrupt_cb.clone(),
             self.vu_common.virtio_common.acked_features,
             backend_req_handler,
             kill_evt,
@@ -300,6 +300,8 @@ impl VirtioDevice for Blk {
             Thread::VirtioVhostBlock,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
         self.vu_common.epoll_thread = Some(epoll_threads.remove(0));

virtio-devices/src/vhost_user/fs.rs

@@ -245,7 +245,7 @@ impl VirtioDevice for Fs {
             mem,
             interrupt_cb,
             queues,
-            ..
+            device_status,
         } = context;
         self.vu_common
             .virtio_common
@@ -260,7 +260,7 @@ impl VirtioDevice for Fs {
         let mut handler = self.vu_common.activate(
             mem,
             &queues,
-            interrupt_cb,
+            interrupt_cb.clone(),
             self.vu_common.virtio_common.acked_features,
             backend_req_handler,
             kill_evt,
@@ -277,6 +277,8 @@ impl VirtioDevice for Fs {
             Thread::VirtioVhostFs,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
         self.vu_common.epoll_thread = Some(epoll_threads.remove(0));

virtio-devices/src/vhost_user/generic_vhost_user.rs

@@ -283,7 +283,7 @@ impl VirtioDevice for GenericVhostUser {
             mem,
             interrupt_cb,
             queues,
-            ..
+            device_status,
         } = context;
         self.vu_common
             .virtio_common
@@ -313,7 +313,7 @@ impl VirtioDevice for GenericVhostUser {
         let mut handler = self.vu_common.activate(
             mem,
             &queues,
-            interrupt_cb,
+            interrupt_cb.clone(),
             self.vu_common.virtio_common.acked_features,
             backend_req_handler,
             kill_evt,
@@ -330,6 +330,8 @@ impl VirtioDevice for GenericVhostUser {
             Thread::VirtioGenericVhostUser,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
         self.vu_common.epoll_thread = Some(epoll_threads.remove(0));

virtio-devices/src/vhost_user/net.rs

@@ -270,7 +270,7 @@ impl VirtioDevice for Net {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.vu_common
             .virtio_common
@@ -321,6 +321,8 @@ impl VirtioDevice for Net {
                 Thread::VirtioVhostNetCtl,
                 &mut epoll_threads,
                 &self.exit_evt,
+                device_status.clone(),
+                interrupt_cb.clone(),
                 move || ctrl_handler.run_ctrl(&paused, paused_sync.as_ref().unwrap()),
             )?;
             self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0));
@@ -340,7 +342,7 @@ impl VirtioDevice for Net {
         let mut handler = self.vu_common.activate(
             mem,
             &queues,
-            interrupt_cb,
+            interrupt_cb.clone(),
             backend_acked_features,
             backend_req_handler,
             kill_evt,
@@ -357,6 +359,8 @@ impl VirtioDevice for Net {
             Thread::VirtioVhostNet,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
         self.vu_common.epoll_thread = Some(epoll_threads.remove(0));

virtio-devices/src/vsock/device.rs

@@ -480,7 +480,7 @@ where
             mem,
             interrupt_cb,
             queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
@@ -498,7 +498,7 @@ where
             queue_evts,
             kill_evt,
             pause_evt,
-            interrupt_cb,
+            interrupt_cb: interrupt_cb.clone(),
             backend: self.backend.clone(),
             access_platform: self.common.access_platform(),
         };
@@ -513,6 +513,8 @@ where
             Thread::VirtioVsock,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;
 

virtio-devices/src/watchdog.rs

@@ -331,7 +331,7 @@ impl VirtioDevice for Watchdog {
             mem,
             interrupt_cb,
             mut queues,
-            ..
+            device_status,
         } = context;
         self.common.activate(&queues, interrupt_cb.clone())?;
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
@@ -351,7 +351,7 @@ impl VirtioDevice for Watchdog {
         let mut handler = WatchdogEpollHandler {
             mem,
             queue,
-            interrupt_cb,
+            interrupt_cb: interrupt_cb.clone(),
             queue_evt,
             kill_evt,
             pause_evt,
@@ -370,6 +370,8 @@ impl VirtioDevice for Watchdog {
             Thread::VirtioWatchdog,
             &mut epoll_threads,
             &self.exit_evt,
+            device_status.clone(),
+            interrupt_cb.clone(),
             move || handler.run(&paused, paused_sync.as_ref().unwrap()),
         )?;