block: raw: Add vectored positioned I/O to AlignedFile

Add read_vectored_at and write_vectored_at to AlignedFile. They take
the aligned fast path with a single preadv or pwritev when the offset
and every iovec base and length satisfy the O_DIRECT alignment, and
otherwise bounce through an AlignedBuffer, scattering on read and
gathering with a read-modify-write on write.

Convert the raw sync engine to these methods and drop its raw preadv
and pwritev block. The methods are unsafe because their soundness
depends on the caller passing iovecs that describe valid memory for
iov_len bytes.

Signed-off-by: Anatol Belski <anbelski@linux.microsoft.com>

block/src/aligned_file.rs

@@ -3,10 +3,10 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::fs::{File, Metadata};
-use std::io;
 use std::os::fd::{AsFd, BorrowedFd};
 use std::os::unix::fs::FileExt;
 use std::os::unix::io::{AsRawFd, RawFd};
+use std::{io, slice};
 
 use vmm_sys_util::file_traits::FileSync;
 use vmm_sys_util::seek_hole::SeekHole;
@@ -24,6 +24,17 @@ fn is_aligned(alignment: usize, buf_ptr: usize, len: usize, offset: u64) -> bool
             && offset.is_multiple_of(alignment as u64))
 }
 
+/// True when `offset` and every iovec base/length satisfy `alignment`
+/// (`alignment == 0` means no O_DIRECT, so everything is "aligned").
+fn iovecs_are_aligned(alignment: usize, iovecs: &[libc::iovec], offset: u64) -> bool {
+    alignment == 0
+        || (offset.is_multiple_of(alignment as u64)
+            && iovecs.iter().all(|iov| {
+                (iov.iov_base as usize).is_multiple_of(alignment)
+                    && iov.iov_len.is_multiple_of(alignment)
+            }))
+}
+
 /// A `File` that transparently satisfies O_DIRECT alignment requirements.
 ///
 /// `alignment == 0` means no O_DIRECT (all I/O passes straight through).
@@ -128,6 +139,92 @@ impl AlignedFile {
         abuf.write_to(&self.file)?;
         Ok(len)
     }
+
+    /// Read into the buffers described by `iovecs`, starting at `offset`.
+    ///
+    /// # Safety
+    /// Every iovec must describe valid, writable memory of `iov_len` bytes.
+    pub(crate) unsafe fn read_vectored_at(
+        &self,
+        iovecs: &[libc::iovec],
+        offset: u64,
+    ) -> io::Result<usize> {
+        if iovecs.is_empty() {
+            return Ok(0);
+        }
+        if iovecs_are_aligned(self.alignment, iovecs, offset) {
+            // SAFETY: upheld by this fn contract.
+            let ret = unsafe {
+                libc::preadv(
+                    self.file.as_raw_fd(),
+                    iovecs.as_ptr(),
+                    iovecs.len() as libc::c_int,
+                    offset as libc::off_t,
+                )
+            };
+            if ret < 0 {
+                return Err(io::Error::last_os_error());
+            }
+            return Ok(ret as usize);
+        }
+        let total_len = iovecs.iter().map(|iov| iov.iov_len).sum();
+        self.read_unaligned(offset, total_len, |mut data| {
+            for iov in iovecs {
+                if data.is_empty() {
+                    break;
+                }
+                let n = data.len().min(iov.iov_len);
+                // SAFETY: upheld by this fn contract.
+                let dst = unsafe { slice::from_raw_parts_mut(iov.iov_base as *mut u8, n) };
+                dst.copy_from_slice(&data[..n]);
+                data = &data[n..];
+            }
+            Ok(())
+        })
+    }
+
+    /// Write the buffers described by `iovecs`, starting at `offset`.
+    ///
+    /// # Safety
+    /// Every iovec must describe valid, readable memory of `iov_len` bytes.
+    pub(crate) unsafe fn write_vectored_at(
+        &self,
+        iovecs: &[libc::iovec],
+        offset: u64,
+    ) -> io::Result<usize> {
+        if iovecs.is_empty() {
+            return Ok(0);
+        }
+        if iovecs_are_aligned(self.alignment, iovecs, offset) {
+            // SAFETY: upheld by this fn contract.
+            let ret = unsafe {
+                libc::pwritev(
+                    self.file.as_raw_fd(),
+                    iovecs.as_ptr(),
+                    iovecs.len() as libc::c_int,
+                    offset as libc::off_t,
+                )
+            };
+            if ret < 0 {
+                return Err(io::Error::last_os_error());
+            }
+            return Ok(ret as usize);
+        }
+        let total_len = iovecs.iter().map(|iov| iov.iov_len).sum();
+        self.write_unaligned(offset, total_len, |mut dst| {
+            for iov in iovecs {
+                if dst.is_empty() {
+                    break;
+                }
+                let n = dst.len().min(iov.iov_len);
+                // SAFETY: upheld by this fn contract.
+                let src = unsafe { slice::from_raw_parts(iov.iov_base as *const u8, n) };
+                dst[..n].copy_from_slice(src);
+                dst = &mut dst[n..];
+            }
+            Ok(())
+        })
+    }
 }
 
 impl FileExt for AlignedFile {

block/src/formats/raw/engine_sync.rs

@@ -10,7 +10,6 @@ use std::os::unix::io::AsRawFd;
 
 use vmm_sys_util::eventfd::EventFd;
 
-use super::{operation_is_aligned, run_unaligned_operation};
 use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
 use crate::sparse::{punch_hole, write_zeroes};
 use crate::{AlignedFile, is_block_device};
@@ -46,52 +45,22 @@ impl AsyncIo for RawSync {
         self.alignment
     }
 
-    fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> {
+    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> {
         let is_read = op.is_read();
+        let iovecs = op.iovecs();
+        let offset = op.offset() as u64;
 
-        let result = if operation_is_aligned(&op, self.alignment) {
-            let fd = self.raw_file.as_raw_fd();
-            let offset = op.offset();
-            let iovecs = op.iovecs();
-
-            let result = if is_read {
-                // SAFETY: the memory pointed to by `iovecs` is backed by the op,
-                // and valid for the kernel to write to by construction of
-                // AsyncIoOperation.
-                unsafe {
-                    libc::preadv(
-                        fd as libc::c_int,
-                        iovecs.as_ptr(),
-                        iovecs.len() as libc::c_int,
-                        offset,
-                    )
-                }
-            } else {
-                // SAFETY: the memory pointed to by `iovecs` is backed by the op,
-                // and valid for the kernel to read from by construction of
-                // AsyncIoOperation.
-                unsafe {
-                    libc::pwritev(
-                        fd as libc::c_int,
-                        iovecs.as_ptr(),
-                        iovecs.len() as libc::c_int,
-                        offset,
-                    )
-                }
-            };
-            if result < 0 {
-                let error = io::Error::last_os_error();
-                return Err(if is_read {
-                    AsyncIoError::ReadVectored(error)
-                } else {
-                    AsyncIoError::WriteVectored(error)
-                });
-            }
-            result as i32
+        let result = if is_read {
+            // SAFETY: op.iovecs() describes valid memory for iov_len bytes by
+            // construction of AsyncIoOperation.
+            unsafe { self.raw_file.read_vectored_at(iovecs, offset) }
+                .map_err(AsyncIoError::ReadVectored)?
         } else {
-            run_unaligned_operation(&self.raw_file, &mut op)?
-        };
-
+            // SAFETY: op.iovecs() describes valid memory for iov_len bytes by
+            // construction of AsyncIoOperation.
+            unsafe { self.raw_file.write_vectored_at(iovecs, offset) }
+                .map_err(AsyncIoError::WriteVectored)?
+        } as i32;
         self.completion_list
             .push_back(AsyncIoCompletion::from_operation(op, result));
         self.eventfd.write(1).unwrap();