From 2359003001bb422ead591e84754494d5d98751d7 Mon Sep 17 00:00:00 2001 From: Anatol Belski Date: Fri, 19 Jun 2026 21:48:33 +0200 Subject: [PATCH] block: raw: Handle O_DIRECT in the raw async workers The raw sync, io_uring and AIO workers now own an AlignedFile and use it for the O_DIRECT alignment value and for the unaligned fallback. Aligned operations keep the fast preadv and pwritev iovec path straight to the kernel. When the offset or an iovec base or length is not a multiple of the probed alignment, the worker gathers the iovecs into one contiguous host buffer and runs a synchronous RMW through AlignedFile, then scatters the result back into guest memory. RawDisk constructs the AlignedFile from the disk file and the direct flag and passes it into each worker, so alignment is probed once at open time. The fixed VHD workers are threaded through the same AlignedFile based constructors using a non-direct AlignedFile to preserve current behavior. Signed-off-by: Anatol Belski --- block/src/factory.rs | 18 ++- block/src/formats/raw/mod.rs | 41 ++++--- block/src/formats/raw/worker/async_aio.rs | 76 ++++++++---- block/src/formats/raw/worker/async_uring.rs | 79 ++++++++---- block/src/formats/raw/worker/mod.rs | 45 +++++++ block/src/formats/raw/worker/sync.rs | 123 +++++++++++-------- block/src/formats/vhd/internal/fixed.rs | 4 + block/src/formats/vhd/mod.rs | 46 ++++--- block/src/formats/vhd/worker/async_uring.rs | 6 +- block/src/formats/vhd/worker/sync.rs | 10 +- fuzz/fuzz_targets/block.rs | 2 +- performance-metrics/src/micro_bench_block.rs | 2 +- 12 files changed, 304 insertions(+), 148 deletions(-) diff --git a/block/src/factory.rs b/block/src/factory.rs index e3518ff1b..6d1de3664 100644 --- a/block/src/factory.rs +++ b/block/src/factory.rs @@ -151,7 +151,11 @@ fn open_raw( if !options.disable_io_uring { if io_uring_supported() { info!("Opening RAW disk file with io_uring backend"); - return Ok(Box::new(RawDisk::new(file, RawBackend::IoUring))); + return Ok(Box::new(RawDisk::new( + file, + RawBackend::IoUring, + options.direct, + ))); } info!("io_uring runtime probe failed for RAW, trying next backend"); } @@ -159,13 +163,21 @@ fn open_raw( if !options.disable_aio { if aio_supported() { info!("Opening RAW disk file with AIO backend"); - return Ok(Box::new(RawDisk::new(file, RawBackend::Aio))); + return Ok(Box::new(RawDisk::new( + file, + RawBackend::Aio, + options.direct, + ))); } info!("AIO runtime probe failed for RAW, using synchronous backend"); } info!("Opening RAW disk file with synchronous backend"); - Ok(Box::new(RawDisk::new(file, RawBackend::Sync))) + Ok(Box::new(RawDisk::new( + file, + RawBackend::Sync, + options.direct, + ))) } fn open_qcow2( diff --git a/block/src/formats/raw/mod.rs b/block/src/formats/raw/mod.rs index adefcee34..c5801a04e 100644 --- a/block/src/formats/raw/mod.rs +++ b/block/src/formats/raw/mod.rs @@ -20,7 +20,7 @@ use self::worker::async_uring::RawAsync; use self::worker::sync::RawSync; use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError}; use crate::error::{BlockError, BlockErrorKind, BlockResult}; -use crate::{DiskTopology, disk_file, probe_sparse_support, query_device_size}; +use crate::{AlignedFile, DiskTopology, disk_file, probe_sparse_support, query_device_size}; pub(crate) mod worker; @@ -46,11 +46,16 @@ pub enum RawBackend { pub struct RawDisk { file: File, backend: RawBackend, + direct: bool, } impl RawDisk { - pub fn new(file: File, backend: RawBackend) -> Self { - Self { file, backend } + pub fn new(file: File, backend: RawBackend, direct: bool) -> Self { + Self { + file, + backend, + direct, + } } } @@ -131,15 +136,21 @@ impl disk_file::AsyncDiskFile for RawDisk { Ok(Box::new(RawDisk { file, backend: self.backend, + direct: self.direct, })) } fn create_async_io(&self, ring_depth: u32) -> BlockResult> { + let file = self + .file + .try_clone() + .map_err(|e| BlockError::new(BlockErrorKind::Io, DiskFileError::Clone(e)))?; + let raw_file = AlignedFile::new(file, self.direct); match self.backend { - RawBackend::Sync => Ok(Box::new(RawSync::new(self.file.as_raw_fd()))), + RawBackend::Sync => Ok(Box::new(RawSync::new(raw_file))), #[cfg(feature = "io_uring")] - RawBackend::IoUring => Ok(Box::new(RawAsync::new(self.file.as_raw_fd(), ring_depth)?)), - RawBackend::Aio => Ok(Box::new(RawAio::new(self.file.as_raw_fd(), ring_depth)?)), + RawBackend::IoUring => Ok(Box::new(RawAsync::new(raw_file, ring_depth)?)), + RawBackend::Aio => Ok(Box::new(RawAio::new(raw_file, ring_depth)?)), } } } @@ -165,7 +176,7 @@ mod unit_tests { #[test] fn new_sync_returns_correct_size() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::Sync); + let disk = RawDisk::new(file, RawBackend::Sync, false); assert_eq!(disk.logical_size().unwrap(), TEST_SIZE); } @@ -201,14 +212,14 @@ mod unit_tests { #[test] fn sync_backend_disables_batch_requests() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::Sync); + let disk = RawDisk::new(file, RawBackend::Sync, false); assert_sync_backend(&disk); } #[test] fn aio_backend_disables_batch_requests() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::Aio); + let disk = RawDisk::new(file, RawBackend::Aio, false); assert_aio_backend(&disk); } @@ -216,7 +227,7 @@ mod unit_tests { #[test] fn io_uring_backend_enables_batch_requests() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::IoUring); + let disk = RawDisk::new(file, RawBackend::IoUring, false); assert_io_uring_backend(&disk); } @@ -228,14 +239,14 @@ mod unit_tests { #[test] fn try_clone_preserves_sync_backend() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::Sync); + let disk = RawDisk::new(file, RawBackend::Sync, false); assert_try_clone(&disk, RawBackend::Sync); } #[test] fn try_clone_preserves_aio_backend() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::Aio); + let disk = RawDisk::new(file, RawBackend::Aio, false); assert_try_clone(&disk, RawBackend::Aio); } @@ -243,14 +254,14 @@ mod unit_tests { #[test] fn try_clone_preserves_io_uring_backend() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::IoUring); + let disk = RawDisk::new(file, RawBackend::IoUring, false); assert_try_clone(&disk, RawBackend::IoUring); } #[test] fn resize_changes_file_size() { let file = make_raw_file(); - let mut disk = RawDisk::new(file, RawBackend::Aio); + let mut disk = RawDisk::new(file, RawBackend::Aio, false); let new_size = TEST_SIZE * 2; disk.resize(new_size).unwrap(); assert_eq!(disk.logical_size().unwrap(), new_size); @@ -259,7 +270,7 @@ mod unit_tests { #[test] fn physical_size_reports_allocated_blocks() { let file = make_raw_file(); - let disk = RawDisk::new(file, RawBackend::Aio); + let disk = RawDisk::new(file, RawBackend::Aio, false); // Sparse file: physical size is less than logical size. assert!(disk.physical_size().unwrap() < disk.logical_size().unwrap()); } diff --git a/block/src/formats/raw/worker/async_aio.rs b/block/src/formats/raw/worker/async_aio.rs index 726a71c76..ff12125d6 100644 --- a/block/src/formats/raw/worker/async_aio.rs +++ b/block/src/formats/raw/worker/async_aio.rs @@ -7,34 +7,36 @@ // Copyright © 2023 Crusoe Energy Systems LLC // -use std::os::unix::io::RawFd; +use std::os::unix::io::AsRawFd; use vmm_sys_util::eventfd::EventFd; +use super::{operation_is_aligned, run_unaligned_operation}; use crate::async_io::{ AioDataIo, AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult, }; use crate::error::{BlockError, BlockErrorKind, BlockResult}; use crate::sparse::{punch_hole, write_zeroes}; -use crate::{SECTOR_SIZE, is_block_device}; +use crate::{AlignedFile, is_block_device}; pub struct RawAio { - fd: RawFd, + raw_file: AlignedFile, data_io: AioDataIo, alignment: u64, is_block_device: bool, } impl RawAio { - pub fn new(fd: RawFd, queue_depth: u32) -> BlockResult { + pub fn new(raw_file: AlignedFile, queue_depth: u32) -> BlockResult { let data_io = AioDataIo::new(queue_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?; - let is_block_device = is_block_device(fd); + let is_block_device = is_block_device(raw_file.as_raw_fd()); + let alignment = raw_file.alignment() as u64; Ok(RawAio { - fd, + raw_file, data_io, - alignment: SECTOR_SIZE, + alignment, is_block_device, }) } @@ -49,25 +51,36 @@ impl AsyncIo for RawAio { self.alignment } - fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> { + fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> { let is_read = op.is_read(); - self.data_io.submit_operation(self.fd, op).map_err(|e| { - if is_read { - AsyncIoError::ReadVectored(e) - } else { - AsyncIoError::WriteVectored(e) - } - }) + + if operation_is_aligned(&op, self.alignment) { + let fd = self.raw_file.as_raw_fd(); + return self.data_io.submit_operation(fd, op).map_err(|e| { + if is_read { + AsyncIoError::ReadVectored(e) + } else { + AsyncIoError::WriteVectored(e) + } + }); + } + + let result = run_unaligned_operation(&self.raw_file, &mut op)?; + self.data_io + .inject_completion(AsyncIoCompletion::from_operation(op, result)); + + Ok(()) } fn fsync(&mut self, user_data: Option) -> AsyncIoResult<()> { + let fd = self.raw_file.as_raw_fd(); if let Some(user_data) = user_data { self.data_io - .submit_fsync(self.fd, user_data) + .submit_fsync(fd, user_data) .map_err(AsyncIoError::Fsync)?; } else { // SAFETY: FFI call with a valid fd - unsafe { libc::fsync(self.fd) }; + unsafe { libc::fsync(fd) }; } Ok(()) @@ -81,8 +94,13 @@ impl AsyncIo for RawAio { // Linux AIO has no IOCB command for fallocate, so perform the // operation synchronously and signal completion via the completion // list, matching the pattern used by the sync backend (RawSync). - punch_hole(self.fd, self.is_block_device, offset, length) - .map_err(AsyncIoError::PunchHole)?; + punch_hole( + self.raw_file.as_raw_fd(), + self.is_block_device, + offset, + length, + ) + .map_err(AsyncIoError::PunchHole)?; self.data_io .inject_completion(AsyncIoCompletion::new(user_data, 0, None)); @@ -91,8 +109,13 @@ impl AsyncIo for RawAio { fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { // Same as punch_hole(). - write_zeroes(self.fd, self.is_block_device, offset, length) - .map_err(AsyncIoError::WriteZeroes)?; + write_zeroes( + self.raw_file.as_raw_fd(), + self.is_block_device, + offset, + length, + ) + .map_err(AsyncIoError::WriteZeroes)?; self.data_io .inject_completion(AsyncIoCompletion::new(user_data, 0, None)); @@ -102,8 +125,6 @@ impl AsyncIo for RawAio { #[cfg(test)] mod unit_tests { - use std::os::unix::io::AsRawFd; - use vmm_sys_util::tempfile::TempFile; use super::*; @@ -113,7 +134,8 @@ mod unit_tests { fn test_punch_hole() { let temp_file = TempFile::new().unwrap(); let mut file = temp_file.into_file(); - let mut async_io = RawAio::new(file.as_raw_fd(), 128).unwrap(); + let mut async_io = + RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap(); tests::test_punch_hole(&mut async_io, &mut file); } @@ -121,7 +143,8 @@ mod unit_tests { fn test_write_zeroes() { let temp_file = TempFile::new().unwrap(); let mut file = temp_file.into_file(); - let mut async_io = RawAio::new(file.as_raw_fd(), 128).unwrap(); + let mut async_io = + RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap(); tests::test_write_zeroes(&mut async_io, &mut file); } @@ -129,7 +152,8 @@ mod unit_tests { fn test_punch_hole_multiple_operations() { let temp_file = TempFile::new().unwrap(); let mut file = temp_file.into_file(); - let mut async_io = RawAio::new(file.as_raw_fd(), 128).unwrap(); + let mut async_io = + RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap(); tests::test_punch_hole_multiple_operations(&mut async_io, &mut file); } } diff --git a/block/src/formats/raw/worker/async_uring.rs b/block/src/formats/raw/worker/async_uring.rs index 3acfac9cd..806e4bd44 100644 --- a/block/src/formats/raw/worker/async_uring.rs +++ b/block/src/formats/raw/worker/async_uring.rs @@ -4,35 +4,37 @@ // // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause -use std::os::unix::io::RawFd; +use std::os::unix::io::AsRawFd; use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE}; use vmm_sys_util::eventfd::EventFd; +use super::{operation_is_aligned, run_unaligned_operation}; use crate::async_io::{ AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult, UringDataIo, }; use crate::error::{BlockError, BlockErrorKind, BlockResult}; use crate::sparse::{blkdiscard, blkzeroout}; -use crate::{SECTOR_SIZE, is_block_device}; +use crate::{AlignedFile, is_block_device}; pub struct RawAsync { - fd: RawFd, + raw_file: AlignedFile, data_io: UringDataIo, alignment: u64, is_block_device: bool, } impl RawAsync { - pub fn new(fd: RawFd, ring_depth: u32) -> BlockResult { + pub fn new(raw_file: AlignedFile, ring_depth: u32) -> BlockResult { let data_io = UringDataIo::new(ring_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?; - let is_block_device = is_block_device(fd); + let is_block_device = is_block_device(raw_file.as_raw_fd()); + let alignment = raw_file.alignment() as u64; Ok(RawAsync { - fd, + raw_file, data_io, - alignment: SECTOR_SIZE, + alignment, is_block_device, }) } @@ -47,25 +49,36 @@ impl AsyncIo for RawAsync { self.alignment } - fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> { + fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> { let is_read = op.is_read(); - self.data_io.submit_operation(self.fd, op).map_err(|e| { - if is_read { - AsyncIoError::ReadVectored(e) - } else { - AsyncIoError::WriteVectored(e) - } - }) + + if operation_is_aligned(&op, self.alignment) { + let fd = self.raw_file.as_raw_fd(); + return self.data_io.submit_operation(fd, op).map_err(|e| { + if is_read { + AsyncIoError::ReadVectored(e) + } else { + AsyncIoError::WriteVectored(e) + } + }); + } + + let result = run_unaligned_operation(&self.raw_file, &mut op)?; + self.data_io + .inject_completion(AsyncIoCompletion::from_operation(op, result)); + + Ok(()) } fn fsync(&mut self, user_data: Option) -> AsyncIoResult<()> { + let fd = self.raw_file.as_raw_fd(); if let Some(user_data) = user_data { self.data_io - .submit_fsync(self.fd, user_data) + .submit_fsync(fd, user_data) .map_err(AsyncIoError::Fsync)?; } else { // SAFETY: FFI call with a valid fd - unsafe { libc::fsync(self.fd) }; + unsafe { libc::fsync(fd) }; } Ok(()) @@ -80,8 +93,28 @@ impl AsyncIo for RawAsync { } fn submit_batch_requests(&mut self, batch_request: Vec) -> AsyncIoResult<()> { + if self.alignment != 0 { + let mut aligned_batch = Vec::with_capacity(batch_request.len()); + for mut op in batch_request { + if operation_is_aligned(&op, self.alignment) { + aligned_batch.push(op); + } else { + let result = run_unaligned_operation(&self.raw_file, &mut op)?; + self.data_io + .inject_completion(AsyncIoCompletion::from_operation(op, result)); + } + } + if aligned_batch.is_empty() { + return Ok(()); + } + return self + .data_io + .submit_batch(self.raw_file.as_raw_fd(), aligned_batch) + .map_err(AsyncIoError::SubmitBatchRequests); + } + self.data_io - .submit_batch(self.fd, batch_request) + .submit_batch(self.raw_file.as_raw_fd(), batch_request) .map_err(AsyncIoError::SubmitBatchRequests) } @@ -91,7 +124,8 @@ impl AsyncIo for RawAsync { // a fallocate request, reaping ENOTSUPP in the completion routine, and reissuing the // request with an ioctl. if self.is_block_device { - blkdiscard(self.fd, offset, length).map_err(AsyncIoError::PunchHole)?; + blkdiscard(self.raw_file.as_raw_fd(), offset, length) + .map_err(AsyncIoError::PunchHole)?; // Deliver the completion through the normal io_uring path by // queuing a NOP carrying `user_data`. The registered eventfd will // fire when it completes, just like any other request. @@ -104,14 +138,15 @@ impl AsyncIo for RawAsync { let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; self.data_io - .submit_fallocate(self.fd, offset, length, mode, user_data) + .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data) .map_err(AsyncIoError::PunchHole) } fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { // Same rationale as punch_hole(). if self.is_block_device { - blkzeroout(self.fd, offset, length).map_err(AsyncIoError::WriteZeroes)?; + blkzeroout(self.raw_file.as_raw_fd(), offset, length) + .map_err(AsyncIoError::WriteZeroes)?; return self .data_io .submit_nop(user_data) @@ -121,7 +156,7 @@ impl AsyncIo for RawAsync { let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE; self.data_io - .submit_fallocate(self.fd, offset, length, mode, user_data) + .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data) .map_err(AsyncIoError::WriteZeroes) } } diff --git a/block/src/formats/raw/worker/mod.rs b/block/src/formats/raw/worker/mod.rs index 9ceb431de..4bbf6eb6c 100644 --- a/block/src/formats/raw/worker/mod.rs +++ b/block/src/formats/raw/worker/mod.rs @@ -7,9 +7,54 @@ //! Each backend implements the [`AsyncIo`](crate::async_io::AsyncIo) //! trait. +use std::os::unix::fs::FileExt; + +use crate::AlignedFile; +use crate::async_io::{AsyncIoError, AsyncIoOperation, AsyncIoResult}; + pub(crate) mod async_aio; #[cfg(feature = "io_uring")] pub(crate) mod async_uring; pub(crate) mod sync; #[cfg(test)] pub(crate) mod tests; + +/// True when `op` satisfies `alignment` and can go straight to the kernel. +pub(crate) fn operation_is_aligned(op: &AsyncIoOperation, alignment: u64) -> bool { + if alignment == 0 { + return true; + } + if !(op.offset() as u64).is_multiple_of(alignment) { + return false; + } + op.iovecs().iter().all(|iov| { + (iov.iov_base as u64).is_multiple_of(alignment) + && (iov.iov_len as u64).is_multiple_of(alignment) + }) +} + +/// Runs an unaligned O_DIRECT operation synchronously through `aligned_file`. +pub(crate) fn run_unaligned_operation( + aligned_file: &AlignedFile, + op: &mut AsyncIoOperation, +) -> AsyncIoResult { + let offset = op.offset() as u64; + let total_len = op.total_len(); + let mut buf = vec![0u8; total_len]; + + if op.is_read() { + let n = aligned_file + .read_at(&mut buf, offset) + .map_err(AsyncIoError::ReadVectored)?; + op.write_bytes_at(0, &buf[..n]) + .map_err(AsyncIoError::ReadVectored)?; + Ok(n as i32) + } else { + op.read_bytes_at(0, &mut buf) + .map_err(AsyncIoError::WriteVectored)?; + let n = aligned_file + .write_at(&buf, offset) + .map_err(AsyncIoError::WriteVectored)?; + Ok(n as i32) + } +} diff --git a/block/src/formats/raw/worker/sync.rs b/block/src/formats/raw/worker/sync.rs index 1bad816bb..4abc82b80 100644 --- a/block/src/formats/raw/worker/sync.rs +++ b/block/src/formats/raw/worker/sync.rs @@ -6,16 +6,17 @@ use std::collections::VecDeque; use std::io; -use std::os::unix::io::RawFd; +use std::os::unix::io::AsRawFd; use vmm_sys_util::eventfd::EventFd; +use super::{operation_is_aligned, run_unaligned_operation}; use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult}; use crate::sparse::{punch_hole, write_zeroes}; -use crate::{SECTOR_SIZE, is_block_device}; +use crate::{AlignedFile, is_block_device}; pub struct RawSync { - fd: RawFd, + raw_file: AlignedFile, eventfd: EventFd, completion_list: VecDeque, alignment: u64, @@ -23,13 +24,14 @@ pub struct RawSync { } impl RawSync { - pub fn new(fd: RawFd) -> Self { - let is_block_device = is_block_device(fd); + pub fn new(raw_file: AlignedFile) -> Self { + let is_block_device = is_block_device(raw_file.as_raw_fd()); + let alignment = raw_file.alignment() as u64; RawSync { - fd, + raw_file, eventfd: EventFd::new(libc::EFD_NONBLOCK).expect("Failed creating EventFd for RawFile"), completion_list: VecDeque::new(), - alignment: SECTOR_SIZE, + alignment, is_block_device, } } @@ -44,47 +46,54 @@ impl AsyncIo for RawSync { self.alignment } - fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> { - let offset = op.offset(); + fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> { let is_read = op.is_read(); - let iovecs = op.iovecs(); - let result = if is_read { - // SAFETY: the memory pointed to by `iovecs` is backed by the op, - // and valid for the kernel to write to by construction of - // AsyncIoOperation. - unsafe { - libc::preadv( - self.fd as libc::c_int, - iovecs.as_ptr(), - iovecs.len() as libc::c_int, - offset, - ) - } - } else { - // SAFETY: the memory pointed to by `iovecs` is backed by the op, - // and valid for the kernel to read from by construction of - // AsyncIoOperation. - unsafe { - libc::pwritev( - self.fd as libc::c_int, - iovecs.as_ptr(), - iovecs.len() as libc::c_int, - offset, - ) - } - }; - if result < 0 { - let error = io::Error::last_os_error(); - return Err(if is_read { - AsyncIoError::ReadVectored(error) + let result = if operation_is_aligned(&op, self.alignment) { + let fd = self.raw_file.as_raw_fd(); + let offset = op.offset(); + let iovecs = op.iovecs(); + + let result = if is_read { + // SAFETY: the memory pointed to by `iovecs` is backed by the op, + // and valid for the kernel to write to by construction of + // AsyncIoOperation. + unsafe { + libc::preadv( + fd as libc::c_int, + iovecs.as_ptr(), + iovecs.len() as libc::c_int, + offset, + ) + } } else { - AsyncIoError::WriteVectored(error) - }); - } + // SAFETY: the memory pointed to by `iovecs` is backed by the op, + // and valid for the kernel to read from by construction of + // AsyncIoOperation. + unsafe { + libc::pwritev( + fd as libc::c_int, + iovecs.as_ptr(), + iovecs.len() as libc::c_int, + offset, + ) + } + }; + if result < 0 { + let error = io::Error::last_os_error(); + return Err(if is_read { + AsyncIoError::ReadVectored(error) + } else { + AsyncIoError::WriteVectored(error) + }); + } + result as i32 + } else { + run_unaligned_operation(&self.raw_file, &mut op)? + }; self.completion_list - .push_back(AsyncIoCompletion::from_operation(op, result as i32)); + .push_back(AsyncIoCompletion::from_operation(op, result)); self.eventfd.write(1).unwrap(); Ok(()) @@ -92,7 +101,7 @@ impl AsyncIo for RawSync { fn fsync(&mut self, user_data: Option) -> AsyncIoResult<()> { // SAFETY: FFI call - let result = unsafe { libc::fsync(self.fd as libc::c_int) }; + let result = unsafe { libc::fsync(self.raw_file.as_raw_fd() as libc::c_int) }; if result < 0 { return Err(AsyncIoError::Fsync(io::Error::last_os_error())); } @@ -111,8 +120,13 @@ impl AsyncIo for RawSync { } fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - punch_hole(self.fd, self.is_block_device, offset, length) - .map_err(AsyncIoError::PunchHole)?; + punch_hole( + self.raw_file.as_raw_fd(), + self.is_block_device, + offset, + length, + ) + .map_err(AsyncIoError::PunchHole)?; self.completion_list .push_back(AsyncIoCompletion::new(user_data, 0, None)); self.eventfd.write(1).unwrap(); @@ -120,8 +134,13 @@ impl AsyncIo for RawSync { } fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - write_zeroes(self.fd, self.is_block_device, offset, length) - .map_err(AsyncIoError::WriteZeroes)?; + write_zeroes( + self.raw_file.as_raw_fd(), + self.is_block_device, + offset, + length, + ) + .map_err(AsyncIoError::WriteZeroes)?; self.completion_list .push_back(AsyncIoCompletion::new(user_data, 0, None)); self.eventfd.write(1).unwrap(); @@ -131,8 +150,6 @@ impl AsyncIo for RawSync { #[cfg(test)] mod unit_tests { - use std::os::unix::io::AsRawFd; - use vmm_sys_util::tempfile::TempFile; use super::*; @@ -142,7 +159,7 @@ mod unit_tests { fn test_punch_hole() { let temp_file = TempFile::new().unwrap(); let mut file = temp_file.into_file(); - let mut async_io = RawSync::new(file.as_raw_fd()); + let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false)); tests::test_punch_hole(&mut async_io, &mut file); } @@ -150,7 +167,7 @@ mod unit_tests { fn test_write_zeroes() { let temp_file = TempFile::new().unwrap(); let mut file = temp_file.into_file(); - let mut async_io = RawSync::new(file.as_raw_fd()); + let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false)); tests::test_write_zeroes(&mut async_io, &mut file); } @@ -158,7 +175,7 @@ mod unit_tests { fn test_punch_hole_multiple_operations() { let temp_file = TempFile::new().unwrap(); let mut file = temp_file.into_file(); - let mut async_io = RawSync::new(file.as_raw_fd()); + let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false)); tests::test_punch_hole_multiple_operations(&mut async_io, &mut file); } } diff --git a/block/src/formats/vhd/internal/fixed.rs b/block/src/formats/vhd/internal/fixed.rs index 3292081d3..e8376e5ec 100644 --- a/block/src/formats/vhd/internal/fixed.rs +++ b/block/src/formats/vhd/internal/fixed.rs @@ -26,6 +26,10 @@ impl FixedVhd { position: 0, }) } + + pub(crate) fn file(&self) -> &File { + &self.file + } } impl AsRawFd for FixedVhd { diff --git a/block/src/formats/vhd/mod.rs b/block/src/formats/vhd/mod.rs index a442a2fc0..71a42f451 100644 --- a/block/src/formats/vhd/mod.rs +++ b/block/src/formats/vhd/mod.rs @@ -25,7 +25,7 @@ use self::worker::sync::FixedVhdSync; use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError}; use crate::disk_file::DiskSize; use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp}; -use crate::{BlockBackend, Error, disk_file}; +use crate::{AlignedFile, BlockBackend, Error, disk_file}; #[derive(Debug)] pub struct VhdDisk { @@ -103,15 +103,15 @@ impl disk_file::AsyncDiskFile for VhdDisk { fn create_async_io(&self, ring_depth: u32) -> BlockResult> { let size = self.logical_size()?; + let file = self.inner.file().try_clone().map_err(|e| { + BlockError::new(BlockErrorKind::Io, DiskFileError::NewAsyncIo(e)).with_op(ErrorOp::Open) + })?; + let raw_file = AlignedFile::new(file, false); if self.use_io_uring { #[cfg(feature = "io_uring")] { - return Ok(Box::new(FixedVhdAsync::new( - self.inner.as_raw_fd(), - ring_depth, - size, - )?)); + return Ok(Box::new(FixedVhdAsync::new(raw_file, ring_depth, size)?)); } #[cfg(not(feature = "io_uring"))] @@ -119,12 +119,7 @@ impl disk_file::AsyncDiskFile for VhdDisk { } let _ = ring_depth; - Ok(Box::new( - FixedVhdSync::new(self.inner.as_raw_fd(), size).map_err(|e| { - BlockError::new(BlockErrorKind::Io, DiskFileError::NewAsyncIo(e)) - .with_op(ErrorOp::Open) - })?, - )) + Ok(Box::new(FixedVhdSync::new(raw_file, size))) } } @@ -132,7 +127,6 @@ impl disk_file::AsyncDiskFile for VhdDisk { mod unit_tests { use std::fs::File; use std::io::{Seek, SeekFrom, Write}; - use std::os::fd::AsRawFd; use vmm_sys_util::tempfile::TempFile; @@ -205,7 +199,8 @@ mod unit_tests { fn sync_rejects_read_straddling_logical_size() { let file = TempFile::new().unwrap().into_file(); file.set_len(0x2000).unwrap(); - let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap(); + let mut sync_io = + FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000); let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1); assert!(matches!( @@ -218,7 +213,8 @@ mod unit_tests { fn sync_rejects_write_straddling_logical_size() { let file = TempFile::new().unwrap().into_file(); file.set_len(0x2000).unwrap(); - let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap(); + let mut sync_io = + FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000); let op = AsyncIoOperation::write_from_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1); @@ -232,7 +228,8 @@ mod unit_tests { fn sync_accepts_operation_exactly_filling_logical_size() { let file = TempFile::new().unwrap().into_file(); file.set_len(0x2000).unwrap(); - let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap(); + let mut sync_io = + FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000); // end == size: boundary must be accepted let op = AsyncIoOperation::read_to_vec(0, OwnedIoBuffer::from_vec(vec![0; 0x1000]), 1); sync_io.submit_data_operation(op).unwrap(); @@ -242,7 +239,8 @@ mod unit_tests { fn sync_accepts_operation_at_last_byte() { let file = TempFile::new().unwrap().into_file(); file.set_len(0x2000).unwrap(); - let mut sync_io = FixedVhdSync::new(file.as_raw_fd(), 0x1000).unwrap(); + let mut sync_io = + FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000); // end = 0xFFF + 1 = 0x1000 == size: boundary must be accepted let op = AsyncIoOperation::read_to_vec(0xFFF, OwnedIoBuffer::from_vec(vec![0; 1]), 1); sync_io.submit_data_operation(op).unwrap(); @@ -253,7 +251,12 @@ mod unit_tests { fn io_uring_batch_rejects_request_straddling_logical_size() { let file = TempFile::new().unwrap().into_file(); file.set_len(0x2000).unwrap(); - let mut async_io = FixedVhdAsync::new(file.as_raw_fd(), 8, 0x1000).unwrap(); + let mut async_io = FixedVhdAsync::new( + AlignedFile::new(file.try_clone().unwrap(), false), + 8, + 0x1000, + ) + .unwrap(); let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1); assert!(matches!( @@ -267,7 +270,12 @@ mod unit_tests { fn io_uring_rejects_single_op_straddling_logical_size() { let file = TempFile::new().unwrap().into_file(); file.set_len(0x2000).unwrap(); - let mut async_io = FixedVhdAsync::new(file.as_raw_fd(), 8, 0x1000).unwrap(); + let mut async_io = FixedVhdAsync::new( + AlignedFile::new(file.try_clone().unwrap(), false), + 8, + 0x1000, + ) + .unwrap(); let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1); assert!(matches!( diff --git a/block/src/formats/vhd/worker/async_uring.rs b/block/src/formats/vhd/worker/async_uring.rs index aa042b470..fd4879fd9 100644 --- a/block/src/formats/vhd/worker/async_uring.rs +++ b/block/src/formats/vhd/worker/async_uring.rs @@ -5,10 +5,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::io; -use std::os::unix::io::RawFd; use vmm_sys_util::eventfd::EventFd; +use crate::AlignedFile; use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult}; use crate::error::BlockResult; use crate::formats::raw::worker::async_uring::RawAsync; @@ -20,8 +20,8 @@ pub struct FixedVhdAsync { } impl FixedVhdAsync { - pub fn new(fd: RawFd, ring_depth: u32, size: u64) -> BlockResult { - let raw_file_async = RawAsync::new(fd, ring_depth)?; + pub fn new(raw_file: AlignedFile, ring_depth: u32, size: u64) -> BlockResult { + let raw_file_async = RawAsync::new(raw_file, ring_depth)?; Ok(FixedVhdAsync { raw_file_async, diff --git a/block/src/formats/vhd/worker/sync.rs b/block/src/formats/vhd/worker/sync.rs index 59aa5bca9..3517239f9 100644 --- a/block/src/formats/vhd/worker/sync.rs +++ b/block/src/formats/vhd/worker/sync.rs @@ -5,10 +5,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::io; -use std::os::unix::io::RawFd; use vmm_sys_util::eventfd::EventFd; +use crate::AlignedFile; use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult}; use crate::formats::raw::worker::sync::RawSync; use crate::formats::vhd::worker::common::validate_operation_bounds; @@ -19,11 +19,11 @@ pub struct FixedVhdSync { } impl FixedVhdSync { - pub fn new(fd: RawFd, size: u64) -> io::Result { - Ok(FixedVhdSync { - raw_file_sync: RawSync::new(fd), + pub fn new(raw_file: AlignedFile, size: u64) -> Self { + FixedVhdSync { + raw_file_sync: RawSync::new(raw_file), size, - }) + } } } diff --git a/fuzz/fuzz_targets/block.rs b/fuzz/fuzz_targets/block.rs index 9a2cde671..d7b1c4120 100644 --- a/fuzz/fuzz_targets/block.rs +++ b/fuzz/fuzz_targets/block.rs @@ -54,7 +54,7 @@ fuzz_target!(|bytes: &[u8]| -> Corpus { let queue_affinity = BTreeMap::new(); let mut block = Block::new( "tmp".to_owned(), - Box::new(RawDisk::new(disk_file, RawBackend::Sync)), + Box::new(RawDisk::new(disk_file, RawBackend::Sync, false)), PathBuf::from(""), false, false, diff --git a/performance-metrics/src/micro_bench_block.rs b/performance-metrics/src/micro_bench_block.rs index 9d13b2def..31ef24b5f 100644 --- a/performance-metrics/src/micro_bench_block.rs +++ b/performance-metrics/src/micro_bench_block.rs @@ -28,7 +28,7 @@ use crate::util::{ pub fn micro_bench_aio_drain(control: &PerformanceTestControl) -> f64 { let num_ops = control.num_ops.expect("num_ops required") as usize; let tmp = util::sized_tempfile(num_ops); - let disk = RawDisk::new(tmp.as_file().try_clone().unwrap(), RawBackend::Aio); + let disk = RawDisk::new(tmp.as_file().try_clone().unwrap(), RawBackend::Aio, false); let mut aio = disk .create_async_io(num_ops as u32) .expect("failed to create AIO context");