From 69563066046609900ca55f4cdf66a5abd4e92f92 Mon Sep 17 00:00:00 2001
From: wuxinyue <wuxinyue.wxy@antgroup.com>
Date: Mon, 19 Aug 2024 14:30:59 +0800
Subject: [PATCH] virtio-devices: block: Reduce notification latency when rate
 limited

When the rate limit was reached it was possible for the notification to
the guest to be lost since the logic to handle the notification was
tightly coupled with processing the queue. The notification would
eventually be triggered when the rate limit pool was refilled but this
could add significant latency.

Address this by refactoring the code to separate processing queue and
signalling - the processing of the queue is suspended when the rate
limit is reached but the signalling will still be attempted if needed
(i.e. VIRTIO_F_EVENT_IDX is still considered.)

Signed-off-by: wuxinyue <wuxinyue.wxy@antgroup.com>
Signed-off-by: Rob Bradford <rbradford@rivosinc.com>
---
 virtio-devices/src/block.rs | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/virtio-devices/src/block.rs b/virtio-devices/src/block.rs
index 8e3e5d6fe..604d6a456 100644
--- a/virtio-devices/src/block.rs
+++ b/virtio-devices/src/block.rs
@@ -235,11 +235,7 @@ impl BlockEpollHandler {
         Ok(())
     }
 
-    fn process_queue_submit_and_signal(&mut self) -> result::Result<(), EpollHelperError> {
-        self.process_queue_submit().map_err(|e| {
-            EpollHelperError::HandleEvent(anyhow!("Failed to process queue (submit): {:?}", e))
-        })?;
-
+    fn try_signal_used_queue(&mut self) -> result::Result<(), EpollHelperError> {
         if self
             .queue
             .needs_notification(self.mem.memory().deref())
@@ -258,6 +254,14 @@ impl BlockEpollHandler {
         Ok(())
     }
 
+    fn process_queue_submit_and_signal(&mut self) -> result::Result<(), EpollHelperError> {
+        self.process_queue_submit().map_err(|e| {
+            EpollHelperError::HandleEvent(anyhow!("Failed to process queue (submit): {:?}", e))
+        })?;
+
+        self.try_signal_used_queue()
+    }
+
     #[inline]
     fn find_inflight_request(&mut self, completed_head: u16) -> Result<Request> {
         // This loop neatly handles the fast path where the completions are
@@ -514,8 +518,14 @@ impl EpollHelperHandler for BlockEpollHandler {
 
                 // Process the queue only when the rate limit is not reached
                 if !rate_limit_reached {
-                    self.process_queue_submit_and_signal()?
+                    self.process_queue_submit().map_err(|e| {
+                        EpollHelperError::HandleEvent(anyhow!(
+                            "Failed to process queue (submit): {:?}",
+                            e
+                        ))
+                    })?;
                 }
+                self.try_signal_used_queue()?;
             }
             RATE_LIMITER_EVENT => {
                 if let Some(rate_limiter) = &mut self.rate_limiter {