vmm: add TLS streams to migration transport

Teach the migration transport to handle TLS-backed streams alongside
plain TCP and UNIX sockets.

Introduce a Tls variant in SocketStream and implement the necessary
traits.

Also updates the local-migration error path to reject any non-UNIX
transport, which now includes TLS-wrapped TCP connections.

On-behalf-of: SAP sebastian.eydam@sap.com
Signed-off-by: Sebastian Eydam <sebastian.eydam@cyberus-technology.de>

vm-migration/src/tls.rs

@@ -18,7 +18,9 @@
 //! other migration streams. All data must pass through rustls; direct I/O on the
 //! underlying socket would bypass TLS processing and break the connection.
 
+use std::io::{self, BufRead, Read, Write};
 use std::net::TcpStream;
+use std::os::fd::{AsFd, BorrowedFd};
 use std::path::Path;
 use std::result;
 use std::sync::Arc;
@@ -31,6 +33,8 @@ use rustls::{
     ClientConfig, ClientConnection, RootCertStore, ServerConfig, ServerConnection, StreamOwned,
 };
 use thiserror::Error;
+use vm_memory::bitmap::BitmapSlice;
+use vm_memory::{ReadVolatile, VolatileMemoryError, VolatileSlice, WriteVolatile};
 
 use crate::MigratableError;
 
@@ -77,9 +81,16 @@ enum TlsStreamParticipant {
 /// Server/Client-agnostic TLS stream.
 pub struct TlsStream {
     stream: TlsStreamParticipant,
+    // `rustls` only accepts plaintext writes as regular byte slices, so
+    // `WriteVolatile` needs a staging buffer to copy out of guest memory.
+    write_buf: Vec<u8>,
 }
 
 impl TlsStream {
+    /// The maximum size of [`TlsStream::write_buf`]. This keeps the reusable buffer
+    /// from growing without bound.
+    const BUF_SIZE: usize = 64 /* KiB */ << 10;
+
     /// Creates a client [`TlsStream`].
     ///
     /// The client verifies the server certificate against `ca-cert.pem` and the
@@ -125,6 +136,7 @@ impl TlsStream {
 
         Ok(Self {
             stream: TlsStreamParticipant::Client(tls),
+            write_buf: Vec::new(),
         })
     }
 
@@ -155,10 +167,110 @@ impl TlsStream {
 
         Ok(Self {
             stream: TlsStreamParticipant::Server(tls),
+            write_buf: Vec::new(),
         })
     }
 }
 
+impl Read for TlsStream {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match &mut self.stream {
+            TlsStreamParticipant::Client(s) => Read::read(s, buf),
+            TlsStreamParticipant::Server(s) => Read::read(s, buf),
+        }
+    }
+}
+
+impl Write for TlsStream {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        match &mut self.stream {
+            TlsStreamParticipant::Client(s) => Write::write(s, buf),
+            TlsStreamParticipant::Server(s) => Write::write(s, buf),
+        }
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        match &mut self.stream {
+            TlsStreamParticipant::Client(s) => Write::flush(s),
+            TlsStreamParticipant::Server(s) => Write::flush(s),
+        }
+    }
+}
+
+// Reading from or writing to these FDs would break the connection, because
+// those reads or writes wouldn't go through rustls. But the FD is necessary to
+// listen for incoming connections.
+impl AsFd for TlsStream {
+    fn as_fd(&self) -> BorrowedFd<'_> {
+        match &self.stream {
+            TlsStreamParticipant::Client(s) => s.get_ref().as_fd(),
+            TlsStreamParticipant::Server(s) => s.get_ref().as_fd(),
+        }
+    }
+}
+
+impl ReadVolatile for TlsStream {
+    fn read_volatile<B: BitmapSlice>(
+        &mut self,
+        vs: &mut VolatileSlice<B>,
+    ) -> result::Result<usize, VolatileMemoryError> {
+        if vs.is_empty() {
+            return Ok(0);
+        }
+
+        let chunk = match &mut self.stream {
+            TlsStreamParticipant::Client(s) => BufRead::fill_buf(s),
+            TlsStreamParticipant::Server(s) => BufRead::fill_buf(s),
+        }
+        .map_err(VolatileMemoryError::IOError)?;
+
+        let n = chunk.len().min(vs.len());
+        if n == 0 {
+            return Ok(0);
+        }
+
+        vs.copy_from(&chunk[..n]);
+
+        match &mut self.stream {
+            TlsStreamParticipant::Client(s) => BufRead::consume(s, n),
+            TlsStreamParticipant::Server(s) => BufRead::consume(s, n),
+        }
+        Ok(n)
+    }
+}
+
+impl WriteVolatile for TlsStream {
+    fn write_volatile<B: BitmapSlice>(
+        &mut self,
+        vs: &VolatileSlice<B>,
+    ) -> Result<usize, VolatileMemoryError> {
+        let len = vs.len().min(Self::BUF_SIZE);
+
+        if len == 0 {
+            return Ok(0);
+        }
+
+        if self.write_buf.len() < len {
+            self.write_buf.resize(len, 0);
+        }
+
+        let buf = &mut self.write_buf[..len];
+        let n = vs.copy_to(buf);
+
+        if n == 0 {
+            return Ok(0);
+        }
+
+        let n = match &mut self.stream {
+            TlsStreamParticipant::Client(s) => Write::write(s, &buf[..n]),
+            TlsStreamParticipant::Server(s) => Write::write(s, &buf[..n]),
+        }
+        .map_err(VolatileMemoryError::IOError)?;
+
+        Ok(n)
+    }
+}
+
 /// Carries a TLS server configuration. Intended to be turned into a [`TlsStream`]
 /// when paired with a [`TcpStream`].
 #[derive(Debug)]

vmm/src/lib.rs

@@ -1470,9 +1470,9 @@ impl Vmm {
                     // Proceed with sending memory file descriptors over UNIX socket
                     vm.send_memory_fds(unix_socket)?;
                 }
-                SocketStream::Tcp(_tcp_socket) => {
+                _ => {
                     return Err(MigratableError::MigrateSend(anyhow!(
-                        "--local option is not supported with TCP sockets",
+                        "--local option is only supported with UNIX sockets",
                     )));
                 }
             }

vmm/src/migration_transport.rs

@@ -26,6 +26,7 @@ use vm_memory::{
     VolatileSlice, WriteVolatile,
 };
 use vm_migration::protocol::{Command, MemoryRangeTable, Request, Response};
+use vm_migration::tls::TlsStream;
 use vm_migration::{MigratableError, Snapshot};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -107,6 +108,7 @@ impl AsFd for ReceiveListener {
 pub(crate) enum SocketStream {
     Unix(UnixStream),
     Tcp(TcpStream),
+    Tls(Box<TlsStream>),
 }
 
 impl Read for SocketStream {
@@ -114,6 +116,7 @@ impl Read for SocketStream {
         match self {
             SocketStream::Unix(stream) => stream.read(buf),
             SocketStream::Tcp(stream) => stream.read(buf),
+            SocketStream::Tls(stream) => stream.read(buf),
         }
     }
 }
@@ -123,6 +126,7 @@ impl Write for SocketStream {
         match self {
             SocketStream::Unix(stream) => stream.write(buf),
             SocketStream::Tcp(stream) => stream.write(buf),
+            SocketStream::Tls(stream) => stream.write(buf),
         }
     }
 
@@ -130,6 +134,7 @@ impl Write for SocketStream {
         match self {
             SocketStream::Unix(stream) => stream.flush(),
             SocketStream::Tcp(stream) => stream.flush(),
+            SocketStream::Tls(stream) => stream.flush(),
         }
     }
 }
@@ -139,6 +144,7 @@ impl AsFd for SocketStream {
         match self {
             SocketStream::Unix(s) => s.as_fd(),
             SocketStream::Tcp(s) => s.as_fd(),
+            SocketStream::Tls(s) => s.as_fd(),
         }
     }
 }
@@ -151,6 +157,7 @@ impl ReadVolatile for SocketStream {
         match self {
             SocketStream::Unix(s) => s.read_volatile(buf),
             SocketStream::Tcp(s) => s.read_volatile(buf),
+            SocketStream::Tls(s) => s.read_volatile(buf),
         }
     }
 }
@@ -163,6 +170,7 @@ impl WriteVolatile for SocketStream {
         match self {
             SocketStream::Unix(s) => s.write_volatile(buf),
             SocketStream::Tcp(s) => s.write_volatile(buf),
+            SocketStream::Tls(s) => s.write_volatile(buf),
         }
     }
 }