block: raw: Handle O_DIRECT in the raw async workers

The raw sync, io_uring and AIO workers now own an AlignedFile and use
it for the O_DIRECT alignment value and for the unaligned fallback.
Aligned operations keep the fast preadv and pwritev iovec path straight
to the kernel. When the offset or an iovec base or length is not a
multiple of the probed alignment, the worker gathers the iovecs into
one contiguous host buffer and runs a synchronous RMW through
AlignedFile, then scatters the result back into guest memory.

RawDisk constructs the AlignedFile from the disk file and the direct
flag and passes it into each worker, so alignment is probed once at
open time. The fixed VHD workers are threaded through the same
AlignedFile based constructors using a non-direct AlignedFile to
preserve current behavior.

Signed-off-by: Anatol Belski <anbelski@linux.microsoft.com>

block/src/factory.rs

@@ -151,7 +151,11 @@ fn open_raw(
     if !options.disable_io_uring {
         if io_uring_supported() {
             info!("Opening RAW disk file with io_uring backend");
-            return Ok(Box::new(RawDisk::new(file, RawBackend::IoUring)));
+            return Ok(Box::new(RawDisk::new(
+                file,
+                RawBackend::IoUring,
+                options.direct,
+            )));
         }
         info!("io_uring runtime probe failed for RAW, trying next backend");
     }
@@ -159,13 +163,21 @@ fn open_raw(
     if !options.disable_aio {
         if aio_supported() {
             info!("Opening RAW disk file with AIO backend");
-            return Ok(Box::new(RawDisk::new(file, RawBackend::Aio)));
+            return Ok(Box::new(RawDisk::new(
+                file,
+                RawBackend::Aio,
+                options.direct,
+            )));
         }
         info!("AIO runtime probe failed for RAW, using synchronous backend");
     }
 
     info!("Opening RAW disk file with synchronous backend");
-    Ok(Box::new(RawDisk::new(file, RawBackend::Sync)))
+    Ok(Box::new(RawDisk::new(
+        file,
+        RawBackend::Sync,
+        options.direct,
+    )))
 }
 
 fn open_qcow2(

block/src/formats/raw/mod.rs

@@ -20,7 +20,7 @@ use self::worker::async_uring::RawAsync;
 use self::worker::sync::RawSync;
 use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError};
 use crate::error::{BlockError, BlockErrorKind, BlockResult};
-use crate::{DiskTopology, disk_file, probe_sparse_support, query_device_size};
+use crate::{AlignedFile, DiskTopology, disk_file, probe_sparse_support, query_device_size};
 
 pub(crate) mod worker;
 
@@ -46,11 +46,16 @@ pub enum RawBackend {
 pub struct RawDisk {
     file: File,
     backend: RawBackend,
+    direct: bool,
 }
 
 impl RawDisk {
-    pub fn new(file: File, backend: RawBackend) -> Self {
-        Self { file, backend }
+    pub fn new(file: File, backend: RawBackend, direct: bool) -> Self {
+        Self {
+            file,
+            backend,
+            direct,
+        }
     }
 }
 
@@ -131,15 +136,21 @@ impl disk_file::AsyncDiskFile for RawDisk {
         Ok(Box::new(RawDisk {
             file,
             backend: self.backend,
+            direct: self.direct,
         }))
     }
 
     fn create_async_io(&self, ring_depth: u32) -> BlockResult<Box<dyn AsyncIo>> {
+        let file = self
+            .file
+            .try_clone()
+            .map_err(|e| BlockError::new(BlockErrorKind::Io, DiskFileError::Clone(e)))?;
+        let raw_file = AlignedFile::new(file, self.direct);
         match self.backend {
-            RawBackend::Sync => Ok(Box::new(RawSync::new(self.file.as_raw_fd()))),
+            RawBackend::Sync => Ok(Box::new(RawSync::new(raw_file))),
             #[cfg(feature = "io_uring")]
-            RawBackend::IoUring => Ok(Box::new(RawAsync::new(self.file.as_raw_fd(), ring_depth)?)),
-            RawBackend::Aio => Ok(Box::new(RawAio::new(self.file.as_raw_fd(), ring_depth)?)),
+            RawBackend::IoUring => Ok(Box::new(RawAsync::new(raw_file, ring_depth)?)),
+            RawBackend::Aio => Ok(Box::new(RawAio::new(raw_file, ring_depth)?)),
         }
     }
 }
@@ -165,7 +176,7 @@ mod unit_tests {
     #[test]
     fn new_sync_returns_correct_size() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Sync);
+        let disk = RawDisk::new(file, RawBackend::Sync, false);
         assert_eq!(disk.logical_size().unwrap(), TEST_SIZE);
     }
 
@@ -201,14 +212,14 @@ mod unit_tests {
     #[test]
     fn sync_backend_disables_batch_requests() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Sync);
+        let disk = RawDisk::new(file, RawBackend::Sync, false);
         assert_sync_backend(&disk);
     }
 
     #[test]
     fn aio_backend_disables_batch_requests() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Aio);
+        let disk = RawDisk::new(file, RawBackend::Aio, false);
         assert_aio_backend(&disk);
     }
 
@@ -216,7 +227,7 @@ mod unit_tests {
     #[test]
     fn io_uring_backend_enables_batch_requests() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::IoUring);
+        let disk = RawDisk::new(file, RawBackend::IoUring, false);
         assert_io_uring_backend(&disk);
     }
 
@@ -228,14 +239,14 @@ mod unit_tests {
     #[test]
     fn try_clone_preserves_sync_backend() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Sync);
+        let disk = RawDisk::new(file, RawBackend::Sync, false);
         assert_try_clone(&disk, RawBackend::Sync);
     }
 
     #[test]
     fn try_clone_preserves_aio_backend() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Aio);
+        let disk = RawDisk::new(file, RawBackend::Aio, false);
         assert_try_clone(&disk, RawBackend::Aio);
     }
 
@@ -243,14 +254,14 @@ mod unit_tests {
     #[test]
     fn try_clone_preserves_io_uring_backend() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::IoUring);
+        let disk = RawDisk::new(file, RawBackend::IoUring, false);
         assert_try_clone(&disk, RawBackend::IoUring);
     }
 
     #[test]
     fn resize_changes_file_size() {
         let file = make_raw_file();
-        let mut disk = RawDisk::new(file, RawBackend::Aio);
+        let mut disk = RawDisk::new(file, RawBackend::Aio, false);
         let new_size = TEST_SIZE * 2;
         disk.resize(new_size).unwrap();
         assert_eq!(disk.logical_size().unwrap(), new_size);
@@ -259,7 +270,7 @@ mod unit_tests {
     #[test]
     fn physical_size_reports_allocated_blocks() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Aio);
+        let disk = RawDisk::new(file, RawBackend::Aio, false);
         // Sparse file: physical size is less than logical size.
         assert!(disk.physical_size().unwrap() < disk.logical_size().unwrap());
     }

block/src/formats/raw/worker/async_aio.rs

@@ -7,34 +7,36 @@
 // Copyright © 2023 Crusoe Energy Systems LLC
 //
 
-use std::os::unix::io::RawFd;
+use std::os::unix::io::AsRawFd;
 
 use vmm_sys_util::eventfd::EventFd;
 
+use super::{operation_is_aligned, run_unaligned_operation};
 use crate::async_io::{
     AioDataIo, AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult,
 };
 use crate::error::{BlockError, BlockErrorKind, BlockResult};
 use crate::sparse::{punch_hole, write_zeroes};
-use crate::{SECTOR_SIZE, is_block_device};
+use crate::{AlignedFile, is_block_device};
 
 pub struct RawAio {
-    fd: RawFd,
+    raw_file: AlignedFile,
     data_io: AioDataIo,
     alignment: u64,
     is_block_device: bool,
 }
 
 impl RawAio {
-    pub fn new(fd: RawFd, queue_depth: u32) -> BlockResult<Self> {
+    pub fn new(raw_file: AlignedFile, queue_depth: u32) -> BlockResult<Self> {
         let data_io =
             AioDataIo::new(queue_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
-        let is_block_device = is_block_device(fd);
+        let is_block_device = is_block_device(raw_file.as_raw_fd());
+        let alignment = raw_file.alignment() as u64;
 
         Ok(RawAio {
-            fd,
+            raw_file,
             data_io,
-            alignment: SECTOR_SIZE,
+            alignment,
             is_block_device,
         })
     }
@@ -49,25 +51,36 @@ impl AsyncIo for RawAio {
         self.alignment
     }
 
-    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> {
+    fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> {
         let is_read = op.is_read();
-        self.data_io.submit_operation(self.fd, op).map_err(|e| {
-            if is_read {
-                AsyncIoError::ReadVectored(e)
-            } else {
-                AsyncIoError::WriteVectored(e)
-            }
-        })
+
+        if operation_is_aligned(&op, self.alignment) {
+            let fd = self.raw_file.as_raw_fd();
+            return self.data_io.submit_operation(fd, op).map_err(|e| {
+                if is_read {
+                    AsyncIoError::ReadVectored(e)
+                } else {
+                    AsyncIoError::WriteVectored(e)
+                }
+            });
+        }
+
+        let result = run_unaligned_operation(&self.raw_file, &mut op)?;
+        self.data_io
+            .inject_completion(AsyncIoCompletion::from_operation(op, result));
+
+        Ok(())
     }
 
     fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        let fd = self.raw_file.as_raw_fd();
         if let Some(user_data) = user_data {
             self.data_io
-                .submit_fsync(self.fd, user_data)
+                .submit_fsync(fd, user_data)
                 .map_err(AsyncIoError::Fsync)?;
         } else {
             // SAFETY: FFI call with a valid fd
-            unsafe { libc::fsync(self.fd) };
+            unsafe { libc::fsync(fd) };
         }
 
         Ok(())
@@ -81,8 +94,13 @@ impl AsyncIo for RawAio {
         // Linux AIO has no IOCB command for fallocate, so perform the
         // operation synchronously and signal completion via the completion
         // list, matching the pattern used by the sync backend (RawSync).
-        punch_hole(self.fd, self.is_block_device, offset, length)
-            .map_err(AsyncIoError::PunchHole)?;
+        punch_hole(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::PunchHole)?;
         self.data_io
             .inject_completion(AsyncIoCompletion::new(user_data, 0, None));
 
@@ -91,8 +109,13 @@ impl AsyncIo for RawAio {
 
     fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
         // Same as punch_hole().
-        write_zeroes(self.fd, self.is_block_device, offset, length)
-            .map_err(AsyncIoError::WriteZeroes)?;
+        write_zeroes(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::WriteZeroes)?;
         self.data_io
             .inject_completion(AsyncIoCompletion::new(user_data, 0, None));
 
@@ -102,8 +125,6 @@ impl AsyncIo for RawAio {
 
 #[cfg(test)]
 mod unit_tests {
-    use std::os::unix::io::AsRawFd;
-
     use vmm_sys_util::tempfile::TempFile;
 
     use super::*;
@@ -113,7 +134,8 @@ mod unit_tests {
     fn test_punch_hole() {
         let temp_file = TempFile::new().unwrap();
         let mut file = temp_file.into_file();
-        let mut async_io = RawAio::new(file.as_raw_fd(), 128).unwrap();
+        let mut async_io =
+            RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap();
         tests::test_punch_hole(&mut async_io, &mut file);
     }
 
@@ -121,7 +143,8 @@ mod unit_tests {
     fn test_write_zeroes() {
         let temp_file = TempFile::new().unwrap();
         let mut file = temp_file.into_file();
-        let mut async_io = RawAio::new(file.as_raw_fd(), 128).unwrap();
+        let mut async_io =
+            RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap();
         tests::test_write_zeroes(&mut async_io, &mut file);
     }
 
@@ -129,7 +152,8 @@ mod unit_tests {
     fn test_punch_hole_multiple_operations() {
         let temp_file = TempFile::new().unwrap();
         let mut file = temp_file.into_file();
-        let mut async_io = RawAio::new(file.as_raw_fd(), 128).unwrap();
+        let mut async_io =
+            RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap();
         tests::test_punch_hole_multiple_operations(&mut async_io, &mut file);
     }
 }

block/src/formats/raw/worker/async_uring.rs

@@ -4,35 +4,37 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
-use std::os::unix::io::RawFd;
+use std::os::unix::io::AsRawFd;
 
 use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE};
 use vmm_sys_util::eventfd::EventFd;
 
+use super::{operation_is_aligned, run_unaligned_operation};
 use crate::async_io::{
     AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult, UringDataIo,
 };
 use crate::error::{BlockError, BlockErrorKind, BlockResult};
 use crate::sparse::{blkdiscard, blkzeroout};
-use crate::{SECTOR_SIZE, is_block_device};
+use crate::{AlignedFile, is_block_device};
 
 pub struct RawAsync {
-    fd: RawFd,
+    raw_file: AlignedFile,
     data_io: UringDataIo,
     alignment: u64,
     is_block_device: bool,
 }
 
 impl RawAsync {
-    pub fn new(fd: RawFd, ring_depth: u32) -> BlockResult<Self> {
+    pub fn new(raw_file: AlignedFile, ring_depth: u32) -> BlockResult<Self> {
         let data_io =
             UringDataIo::new(ring_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
-        let is_block_device = is_block_device(fd);
+        let is_block_device = is_block_device(raw_file.as_raw_fd());
+        let alignment = raw_file.alignment() as u64;
 
         Ok(RawAsync {
-            fd,
+            raw_file,
             data_io,
-            alignment: SECTOR_SIZE,
+            alignment,
             is_block_device,
         })
     }
@@ -47,25 +49,36 @@ impl AsyncIo for RawAsync {
         self.alignment
     }
 
-    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> {
+    fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> {
         let is_read = op.is_read();
-        self.data_io.submit_operation(self.fd, op).map_err(|e| {
-            if is_read {
-                AsyncIoError::ReadVectored(e)
-            } else {
-                AsyncIoError::WriteVectored(e)
-            }
-        })
+
+        if operation_is_aligned(&op, self.alignment) {
+            let fd = self.raw_file.as_raw_fd();
+            return self.data_io.submit_operation(fd, op).map_err(|e| {
+                if is_read {
+                    AsyncIoError::ReadVectored(e)
+                } else {
+                    AsyncIoError::WriteVectored(e)
+                }
+            });
+        }
+
+        let result = run_unaligned_operation(&self.raw_file, &mut op)?;
+        self.data_io
+            .inject_completion(AsyncIoCompletion::from_operation(op, result));
+
+        Ok(())
     }
 
     fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        let fd = self.raw_file.as_raw_fd();
         if let Some(user_data) = user_data {
             self.data_io
-                .submit_fsync(self.fd, user_data)
+                .submit_fsync(fd, user_data)
                 .map_err(AsyncIoError::Fsync)?;
         } else {
             // SAFETY: FFI call with a valid fd
-            unsafe { libc::fsync(self.fd) };
+            unsafe { libc::fsync(fd) };
         }
 
         Ok(())
@@ -80,8 +93,28 @@ impl AsyncIo for RawAsync {
     }
 
     fn submit_batch_requests(&mut self, batch_request: Vec<AsyncIoOperation>) -> AsyncIoResult<()> {
+        if self.alignment != 0 {
+            let mut aligned_batch = Vec::with_capacity(batch_request.len());
+            for mut op in batch_request {
+                if operation_is_aligned(&op, self.alignment) {
+                    aligned_batch.push(op);
+                } else {
+                    let result = run_unaligned_operation(&self.raw_file, &mut op)?;
+                    self.data_io
+                        .inject_completion(AsyncIoCompletion::from_operation(op, result));
+                }
+            }
+            if aligned_batch.is_empty() {
+                return Ok(());
+            }
+            return self
+                .data_io
+                .submit_batch(self.raw_file.as_raw_fd(), aligned_batch)
+                .map_err(AsyncIoError::SubmitBatchRequests);
+        }
+
         self.data_io
-            .submit_batch(self.fd, batch_request)
+            .submit_batch(self.raw_file.as_raw_fd(), batch_request)
             .map_err(AsyncIoError::SubmitBatchRequests)
     }
 
@@ -91,7 +124,8 @@ impl AsyncIo for RawAsync {
         // a fallocate request, reaping ENOTSUPP in the completion routine, and reissuing the
         // request with an ioctl.
         if self.is_block_device {
-            blkdiscard(self.fd, offset, length).map_err(AsyncIoError::PunchHole)?;
+            blkdiscard(self.raw_file.as_raw_fd(), offset, length)
+                .map_err(AsyncIoError::PunchHole)?;
             // Deliver the completion through the normal io_uring path by
             // queuing a NOP carrying `user_data`. The registered eventfd will
             // fire when it completes, just like any other request.
@@ -104,14 +138,15 @@ impl AsyncIo for RawAsync {
         let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
 
         self.data_io
-            .submit_fallocate(self.fd, offset, length, mode, user_data)
+            .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data)
             .map_err(AsyncIoError::PunchHole)
     }
 
     fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
         // Same rationale as punch_hole().
         if self.is_block_device {
-            blkzeroout(self.fd, offset, length).map_err(AsyncIoError::WriteZeroes)?;
+            blkzeroout(self.raw_file.as_raw_fd(), offset, length)
+                .map_err(AsyncIoError::WriteZeroes)?;
             return self
                 .data_io
                 .submit_nop(user_data)
@@ -121,7 +156,7 @@ impl AsyncIo for RawAsync {
         let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
 
         self.data_io
-            .submit_fallocate(self.fd, offset, length, mode, user_data)
+            .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data)
             .map_err(AsyncIoError::WriteZeroes)
     }
 }

block/src/formats/raw/worker/mod.rs

@@ -7,9 +7,54 @@
 //! Each backend implements the [`AsyncIo`](crate::async_io::AsyncIo)
 //! trait.
 
+use std::os::unix::fs::FileExt;
+
+use crate::AlignedFile;
+use crate::async_io::{AsyncIoError, AsyncIoOperation, AsyncIoResult};
+
 pub(crate) mod async_aio;
 #[cfg(feature = "io_uring")]
 pub(crate) mod async_uring;
 pub(crate) mod sync;
 #[cfg(test)]
 pub(crate) mod tests;
+
+/// True when `op` satisfies `alignment` and can go straight to the kernel.
+pub(crate) fn operation_is_aligned(op: &AsyncIoOperation, alignment: u64) -> bool {
+    if alignment == 0 {
+        return true;
+    }
+    if !(op.offset() as u64).is_multiple_of(alignment) {
+        return false;
+    }
+    op.iovecs().iter().all(|iov| {
+        (iov.iov_base as u64).is_multiple_of(alignment)
+            && (iov.iov_len as u64).is_multiple_of(alignment)
+    })
+}
+
+/// Runs an unaligned O_DIRECT operation synchronously through `aligned_file`.
+pub(crate) fn run_unaligned_operation(
+    aligned_file: &AlignedFile,
+    op: &mut AsyncIoOperation,
+) -> AsyncIoResult<i32> {
+    let offset = op.offset() as u64;
+    let total_len = op.total_len();
+    let mut buf = vec![0u8; total_len];
+
+    if op.is_read() {
+        let n = aligned_file
+            .read_at(&mut buf, offset)
+            .map_err(AsyncIoError::ReadVectored)?;
+        op.write_bytes_at(0, &buf[..n])
+            .map_err(AsyncIoError::ReadVectored)?;
+        Ok(n as i32)
+    } else {
+        op.read_bytes_at(0, &mut buf)
+            .map_err(AsyncIoError::WriteVectored)?;
+        let n = aligned_file
+            .write_at(&buf, offset)
+            .map_err(AsyncIoError::WriteVectored)?;
+        Ok(n as i32)
+    }
+}

block/src/formats/raw/worker/sync.rs

@@ -6,16 +6,17 @@
 
 use std::collections::VecDeque;
 use std::io;
-use std::os::unix::io::RawFd;
+use std::os::unix::io::AsRawFd;
 
 use vmm_sys_util::eventfd::EventFd;
 
+use super::{operation_is_aligned, run_unaligned_operation};
 use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
 use crate::sparse::{punch_hole, write_zeroes};
-use crate::{SECTOR_SIZE, is_block_device};
+use crate::{AlignedFile, is_block_device};
 
 pub struct RawSync {
-    fd: RawFd,
+    raw_file: AlignedFile,
     eventfd: EventFd,
     completion_list: VecDeque<AsyncIoCompletion>,
     alignment: u64,
@@ -23,13 +24,14 @@ pub struct RawSync {
 }
 
 impl RawSync {
-    pub fn new(fd: RawFd) -> Self {
-        let is_block_device = is_block_device(fd);
+    pub fn new(raw_file: AlignedFile) -> Self {
+        let is_block_device = is_block_device(raw_file.as_raw_fd());
+        let alignment = raw_file.alignment() as u64;
         RawSync {
-            fd,
+            raw_file,
             eventfd: EventFd::new(libc::EFD_NONBLOCK).expect("Failed creating EventFd for RawFile"),
             completion_list: VecDeque::new(),
-            alignment: SECTOR_SIZE,
+            alignment,
             is_block_device,
         }
     }
@@ -44,47 +46,54 @@ impl AsyncIo for RawSync {
         self.alignment
     }
 
-    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> {
-        let offset = op.offset();
+    fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> {
         let is_read = op.is_read();
-        let iovecs = op.iovecs();
 
-        let result = if is_read {
-            // SAFETY: the memory pointed to by `iovecs` is backed by the op,
-            // and valid for the kernel to write to by construction of
-            // AsyncIoOperation.
-            unsafe {
-                libc::preadv(
-                    self.fd as libc::c_int,
-                    iovecs.as_ptr(),
-                    iovecs.len() as libc::c_int,
-                    offset,
-                )
-            }
-        } else {
-            // SAFETY: the memory pointed to by `iovecs` is backed by the op,
-            // and valid for the kernel to read from by construction of
-            // AsyncIoOperation.
-            unsafe {
-                libc::pwritev(
-                    self.fd as libc::c_int,
-                    iovecs.as_ptr(),
-                    iovecs.len() as libc::c_int,
-                    offset,
-                )
-            }
-        };
-        if result < 0 {
-            let error = io::Error::last_os_error();
-            return Err(if is_read {
-                AsyncIoError::ReadVectored(error)
+        let result = if operation_is_aligned(&op, self.alignment) {
+            let fd = self.raw_file.as_raw_fd();
+            let offset = op.offset();
+            let iovecs = op.iovecs();
+
+            let result = if is_read {
+                // SAFETY: the memory pointed to by `iovecs` is backed by the op,
+                // and valid for the kernel to write to by construction of
+                // AsyncIoOperation.
+                unsafe {
+                    libc::preadv(
+                        fd as libc::c_int,
+                        iovecs.as_ptr(),
+                        iovecs.len() as libc::c_int,
+                        offset,
+                    )
+                }
             } else {
-                AsyncIoError::WriteVectored(error)
-            });
-        }
+                // SAFETY: the memory pointed to by `iovecs` is backed by the op,
+                // and valid for the kernel to read from by construction of
+                // AsyncIoOperation.
+                unsafe {
+                    libc::pwritev(
+                        fd as libc::c_int,
+                        iovecs.as_ptr(),
+                        iovecs.len() as libc::c_int,
+                        offset,
+                    )
+                }
+            };
+            if result < 0 {
+                let error = io::Error::last_os_error();
+                return Err(if is_read {
+                    AsyncIoError::ReadVectored(error)
+                } else {
+                    AsyncIoError::WriteVectored(error)
+                });
+            }
+            result as i32
+        } else {
+            run_unaligned_operation(&self.raw_file, &mut op)?
+        };
 
         self.completion_list
-            .push_back(AsyncIoCompletion::from_operation(op, result as i32));
+            .push_back(AsyncIoCompletion::from_operation(op, result));
         self.eventfd.write(1).unwrap();
 
         Ok(())
@@ -92,7 +101,7 @@ impl AsyncIo for RawSync {
 
     fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
         // SAFETY: FFI call
-        let result = unsafe { libc::fsync(self.fd as libc::c_int) };
+        let result = unsafe { libc::fsync(self.raw_file.as_raw_fd() as libc::c_int) };
         if result < 0 {
             return Err(AsyncIoError::Fsync(io::Error::last_os_error()));
         }
@@ -111,8 +120,13 @@ impl AsyncIo for RawSync {
     }
 
     fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        punch_hole(self.fd, self.is_block_device, offset, length)
-            .map_err(AsyncIoError::PunchHole)?;
+        punch_hole(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::PunchHole)?;
         self.completion_list
             .push_back(AsyncIoCompletion::new(user_data, 0, None));
         self.eventfd.write(1).unwrap();
@@ -120,8 +134,13 @@ impl AsyncIo for RawSync {
     }
 
     fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        write_zeroes(self.fd, self.is_block_device, offset, length)
-            .map_err(AsyncIoError::WriteZeroes)?;
+        write_zeroes(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::WriteZeroes)?;
         self.completion_list
             .push_back(AsyncIoCompletion::new(user_data, 0, None));
         self.eventfd.write(1).unwrap();
@@ -131,8 +150,6 @@ impl AsyncIo for RawSync {
 
 #[cfg(test)]
 mod unit_tests {
-    use std::os::unix::io::AsRawFd;
-
     use vmm_sys_util::tempfile::TempFile;
 
     use super::*;
@@ -142,7 +159,7 @@ mod unit_tests {
     fn test_punch_hole() {
         let temp_file = TempFile::new().unwrap();
         let mut file = temp_file.into_file();
-        let mut async_io = RawSync::new(file.as_raw_fd());
+        let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false));
         tests::test_punch_hole(&mut async_io, &mut file);
     }
 
@@ -150,7 +167,7 @@ mod unit_tests {
     fn test_write_zeroes() {
         let temp_file = TempFile::new().unwrap();
         let mut file = temp_file.into_file();
-        let mut async_io = RawSync::new(file.as_raw_fd());
+        let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false));
         tests::test_write_zeroes(&mut async_io, &mut file);
     }
 
@@ -158,7 +175,7 @@ mod unit_tests {
     fn test_punch_hole_multiple_operations() {
         let temp_file = TempFile::new().unwrap();
         let mut file = temp_file.into_file();
-        let mut async_io = RawSync::new(file.as_raw_fd());
+        let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false));
         tests::test_punch_hole_multiple_operations(&mut async_io, &mut file);
     }
 }

block/src/formats/vhd/internal/fixed.rs

@@ -26,6 +26,10 @@ impl FixedVhd {
             position: 0,
         })
     }
+
+    pub(crate) fn file(&self) -> &File {
+        &self.file
+    }
 }
 
 impl AsRawFd for FixedVhd {

block/src/formats/vhd/mod.rs

@@ -25,7 +25,7 @@ use self::worker::sync::FixedVhdSync;
 use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError};
 use crate::disk_file::DiskSize;
 use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp};
-use crate::{BlockBackend, Error, disk_file};
+use crate::{AlignedFile, BlockBackend, Error, disk_file};
 
 #[derive(Debug)]
 pub struct VhdDisk {
@@ -103,15 +103,15 @@ impl disk_file::AsyncDiskFile for VhdDisk {
 
     fn create_async_io(&self, ring_depth: u32) -> BlockResult<Box<dyn AsyncIo>> {
         let size = self.logical_size()?;
+        let file = self.inner.file().try_clone().map_err(|e| {
+            BlockError::new(BlockErrorKind::Io, DiskFileError::NewAsyncIo(e)).with_op(ErrorOp::Open)
+        })?;
+        let raw_file = AlignedFile::new(file, false);
 
         if self.use_io_uring {
             #[cfg(feature = "io_uring")]
             {
-                return Ok(Box::new(FixedVhdAsync::new(
-                    self.inner.as_raw_fd(),
-                    ring_depth,
-                    size,
-                )?));
+                return Ok(Box::new(FixedVhdAsync::new(raw_file, ring_depth, size)?));
             }
 
             #[cfg(not(feature = "io_uring"))]
@@ -119,12 +119,7 @@ impl disk_file::AsyncDiskFile for VhdDisk {
         }
 
         let _ = ring_depth;
-        Ok(Box::new(
-            FixedVhdSync::new(self.inner.as_raw_fd(), size).map_err(|e| {
-                BlockError::new(BlockErrorKind::Io, DiskFileError::NewAsyncIo(e))
-                    .with_op(ErrorOp::Open)
-            })?,
-        ))
+        Ok(Box::new(FixedVhdSync::new(raw_file, size)))
     }
 }
 
@@ -132,7 +127,6 @@ impl disk_file::AsyncDiskFile for VhdDisk {
 mod unit_tests {
     use std::fs::File;
     use std::io::{Seek, SeekFrom, Write};
-    use std::os::fd::AsRawFd;
 
     use vmm_sys_util::tempfile::TempFile;
 
@@ -205,7 +199,8 @@ mod unit_tests {
     fn sync_rejects_read_straddling_logical_size() {
         let file = TempFile::new().unwrap().into_file();
         file.set_len(0x2000).unwrap();
-        let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
         let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
 
         assert!(matches!(
@@ -218,7 +213,8 @@ mod unit_tests {
     fn sync_rejects_write_straddling_logical_size() {
         let file = TempFile::new().unwrap().into_file();
         file.set_len(0x2000).unwrap();
-        let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
         let op =
             AsyncIoOperation::write_from_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
 
@@ -232,7 +228,8 @@ mod unit_tests {
     fn sync_accepts_operation_exactly_filling_logical_size() {
         let file = TempFile::new().unwrap().into_file();
         file.set_len(0x2000).unwrap();
-        let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
         // end == size: boundary must be accepted
         let op = AsyncIoOperation::read_to_vec(0, OwnedIoBuffer::from_vec(vec![0; 0x1000]), 1);
         sync_io.submit_data_operation(op).unwrap();
@@ -242,7 +239,8 @@ mod unit_tests {
     fn sync_accepts_operation_at_last_byte() {
         let file = TempFile::new().unwrap().into_file();
         file.set_len(0x2000).unwrap();
-        let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
         // end = 0xFFF + 1 = 0x1000 == size: boundary must be accepted
         let op = AsyncIoOperation::read_to_vec(0xFFF, OwnedIoBuffer::from_vec(vec![0; 1]), 1);
         sync_io.submit_data_operation(op).unwrap();
@@ -253,7 +251,12 @@ mod unit_tests {
     fn io_uring_batch_rejects_request_straddling_logical_size() {
         let file = TempFile::new().unwrap().into_file();
         file.set_len(0x2000).unwrap();
-        let mut async_io = FixedVhdAsync::new(file.as_raw_fd(), 8, 0x1000).unwrap();
+        let mut async_io = FixedVhdAsync::new(
+            AlignedFile::new(file.try_clone().unwrap(), false),
+            8,
+            0x1000,
+        )
+        .unwrap();
         let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
 
         assert!(matches!(
@@ -267,7 +270,12 @@ mod unit_tests {
     fn io_uring_rejects_single_op_straddling_logical_size() {
         let file = TempFile::new().unwrap().into_file();
         file.set_len(0x2000).unwrap();
-        let mut async_io = FixedVhdAsync::new(file.as_raw_fd(), 8, 0x1000).unwrap();
+        let mut async_io = FixedVhdAsync::new(
+            AlignedFile::new(file.try_clone().unwrap(), false),
+            8,
+            0x1000,
+        )
+        .unwrap();
         let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
 
         assert!(matches!(

block/src/formats/vhd/worker/async_uring.rs

@@ -5,10 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::io;
-use std::os::unix::io::RawFd;
 
 use vmm_sys_util::eventfd::EventFd;
 
+use crate::AlignedFile;
 use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
 use crate::error::BlockResult;
 use crate::formats::raw::worker::async_uring::RawAsync;
@@ -20,8 +20,8 @@ pub struct FixedVhdAsync {
 }
 
 impl FixedVhdAsync {
-    pub fn new(fd: RawFd, ring_depth: u32, size: u64) -> BlockResult<Self> {
-        let raw_file_async = RawAsync::new(fd, ring_depth)?;
+    pub fn new(raw_file: AlignedFile, ring_depth: u32, size: u64) -> BlockResult<Self> {
+        let raw_file_async = RawAsync::new(raw_file, ring_depth)?;
 
         Ok(FixedVhdAsync {
             raw_file_async,

block/src/formats/vhd/worker/sync.rs

@@ -5,10 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::io;
-use std::os::unix::io::RawFd;
 
 use vmm_sys_util::eventfd::EventFd;
 
+use crate::AlignedFile;
 use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
 use crate::formats::raw::worker::sync::RawSync;
 use crate::formats::vhd::worker::common::validate_operation_bounds;
@@ -19,11 +19,11 @@ pub struct FixedVhdSync {
 }
 
 impl FixedVhdSync {
-    pub fn new(fd: RawFd, size: u64) -> io::Result<Self> {
-        Ok(FixedVhdSync {
-            raw_file_sync: RawSync::new(fd),
+    pub fn new(raw_file: AlignedFile, size: u64) -> Self {
+        FixedVhdSync {
+            raw_file_sync: RawSync::new(raw_file),
             size,
-        })
+        }
     }
 }
 

fuzz/fuzz_targets/block.rs

@@ -54,7 +54,7 @@ fuzz_target!(|bytes: &[u8]| -> Corpus {
     let queue_affinity = BTreeMap::new();
     let mut block = Block::new(
         "tmp".to_owned(),
-        Box::new(RawDisk::new(disk_file, RawBackend::Sync)),
+        Box::new(RawDisk::new(disk_file, RawBackend::Sync, false)),
         PathBuf::from(""),
         false,
         false,

performance-metrics/src/micro_bench_block.rs

@@ -28,7 +28,7 @@ use crate::util::{
 pub fn micro_bench_aio_drain(control: &PerformanceTestControl) -> f64 {
     let num_ops = control.num_ops.expect("num_ops required") as usize;
     let tmp = util::sized_tempfile(num_ops);
-    let disk = RawDisk::new(tmp.as_file().try_clone().unwrap(), RawBackend::Aio);
+    let disk = RawDisk::new(tmp.as_file().try_clone().unwrap(), RawBackend::Aio, false);
     let mut aio = disk
         .create_async_io(num_ops as u32)
         .expect("failed to create AIO context");