vmm: Refactor serial buffer to allow flush on PTY when writable

Refactor the serial buffer handling in order to write the serial
buffer's output to a PTY connected after the serial device stops being
written to by the guest.

This change moves the serial buffer initialization inside the serial
manager. That is done to allow the serial buffer to be made aware of
the PTY and epoll fds needed in order to modify the
EpollDispatch::File trigger. These are then used by the serial buffer
to trigger an epoll event when the PTY fd is writable and the buffer
has content in it. They are also used to remove the trigger when the
buffer is emptied in order to avoid unnecessary wake-ups.

Signed-off-by: William Douglas <william.douglas@intel.com>

vmm/src/serial_buffer.rs

@@ -3,7 +3,10 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use crate::serial_manager::EpollDispatch;
+
 use std::io::Write;
+use std::os::unix::io::RawFd;
 
 // Circular buffer implementation for serial output.
 // Read from head; push to tail
@@ -12,6 +15,9 @@ pub(crate) struct SerialBuffer {
     head: usize,
     tail: usize,
     out: Box<dyn Write + Send>,
+    buffering: bool,
+    out_fd: Option<RawFd>,
+    epoll_fd: Option<RawFd>,
 }
 
 const MAX_BUFFER_SIZE: usize = 16 << 10;
@@ -23,10 +29,21 @@ impl SerialBuffer {
             head: 0,
             tail: 0,
             out,
+            buffering: false,
+            out_fd: None,
+            epoll_fd: None,
         }
     }
 
-    pub(crate) fn flush_buffer(&mut self) -> Result<(), std::io::Error> {
+    pub(crate) fn add_out_fd(&mut self, out_fd: RawFd) {
+        self.out_fd = Some(out_fd);
+    }
+
+    pub(crate) fn add_epoll_fd(&mut self, epoll_fd: RawFd) {
+        self.epoll_fd = Some(epoll_fd);
+    }
+
+    pub fn flush_buffer(&mut self) -> Result<(), std::io::Error> {
         if self.tail <= self.head {
             // The buffer to be written is in two parts
             let buf = &self.buffer[self.head..];
@@ -45,6 +62,7 @@ impl SerialBuffer {
                     if !matches!(e.kind(), std::io::ErrorKind::WouldBlock) {
                         return Err(e);
                     }
+                    self.add_out_poll()?;
                     return Ok(());
                 }
             }
@@ -58,6 +76,7 @@ impl SerialBuffer {
                     self.buffer.shrink_to_fit();
                     self.head = 0;
                     self.tail = 0;
+                    self.remove_out_poll()?;
                 } else {
                     self.head += bytes_written;
                 }
@@ -67,11 +86,45 @@ impl SerialBuffer {
                 if !matches!(e.kind(), std::io::ErrorKind::WouldBlock) {
                     return Err(e);
                 }
+                self.add_out_poll()?;
             }
         }
 
         Ok(())
     }
+
+    fn add_out_poll(&mut self) -> Result<(), std::io::Error> {
+        if self.out_fd.is_some() && self.epoll_fd.is_some() && !self.buffering {
+            self.buffering = true;
+            let out_fd = self.out_fd.as_ref().unwrap();
+            let epoll_fd = self.epoll_fd.as_ref().unwrap();
+            epoll::ctl(
+                *epoll_fd,
+                epoll::ControlOptions::EPOLL_CTL_MOD,
+                *out_fd,
+                epoll::Event::new(
+                    epoll::Events::EPOLLIN | epoll::Events::EPOLLOUT,
+                    EpollDispatch::File as u64,
+                ),
+            )?;
+        }
+        Ok(())
+    }
+
+    fn remove_out_poll(&mut self) -> Result<(), std::io::Error> {
+        if self.out_fd.is_some() && self.epoll_fd.is_some() && self.buffering {
+            self.buffering = false;
+            let out_fd = self.out_fd.as_ref().unwrap();
+            let epoll_fd = self.epoll_fd.as_ref().unwrap();
+            epoll::ctl(
+                *epoll_fd,
+                epoll::ControlOptions::EPOLL_CTL_MOD,
+                *out_fd,
+                epoll::Event::new(epoll::Events::EPOLLIN, EpollDispatch::File as u64),
+            )?;
+        }
+        Ok(())
+    }
 }
 impl Write for SerialBuffer {
     fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
@@ -83,6 +136,7 @@ impl Write for SerialBuffer {
                     if !matches!(e.kind(), std::io::ErrorKind::WouldBlock) {
                         return Err(e);
                     }
+                    self.add_out_poll()?;
                     self.buffer.push(*v);
                     self.tail += 1;
                 } else {
@@ -116,6 +170,6 @@ impl Write for SerialBuffer {
     }
 
     fn flush(&mut self) -> Result<(), std::io::Error> {
-        Ok(())
+        self.flush_buffer()
     }
 }