From 206cb1e100decdd091c6cef17e94a8814d6335b4 Mon Sep 17 00:00:00 2001 From: Anatol Belski Date: Sun, 19 Jul 2026 23:24:43 +0200 Subject: [PATCH] block: raw: Add vectored positioned I/O to AlignedFile Add read_vectored_at and write_vectored_at to AlignedFile. They take the aligned fast path with a single preadv or pwritev when the offset and every iovec base and length satisfy the O_DIRECT alignment, and otherwise bounce through an AlignedBuffer, scattering on read and gathering with a read-modify-write on write. Convert the raw sync engine to these methods and drop its raw preadv and pwritev block. The methods are unsafe because their soundness depends on the caller passing iovecs that describe valid memory for iov_len bytes. Signed-off-by: Anatol Belski --- block/src/aligned_file.rs | 99 +++++++++++++++++++++++++++- block/src/formats/raw/engine_sync.rs | 57 ++++------------ 2 files changed, 111 insertions(+), 45 deletions(-) diff --git a/block/src/aligned_file.rs b/block/src/aligned_file.rs index 1dd919147..67a96efed 100644 --- a/block/src/aligned_file.rs +++ b/block/src/aligned_file.rs @@ -3,10 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::fs::{File, Metadata}; -use std::io; use std::os::fd::{AsFd, BorrowedFd}; use std::os::unix::fs::FileExt; use std::os::unix::io::{AsRawFd, RawFd}; +use std::{io, slice}; use vmm_sys_util::file_traits::FileSync; use vmm_sys_util::seek_hole::SeekHole; @@ -24,6 +24,17 @@ fn is_aligned(alignment: usize, buf_ptr: usize, len: usize, offset: u64) -> bool && offset.is_multiple_of(alignment as u64)) } +/// True when `offset` and every iovec base/length satisfy `alignment` +/// (`alignment == 0` means no O_DIRECT, so everything is "aligned"). +fn iovecs_are_aligned(alignment: usize, iovecs: &[libc::iovec], offset: u64) -> bool { + alignment == 0 + || (offset.is_multiple_of(alignment as u64) + && iovecs.iter().all(|iov| { + (iov.iov_base as usize).is_multiple_of(alignment) + && iov.iov_len.is_multiple_of(alignment) + })) +} + /// A `File` that transparently satisfies O_DIRECT alignment requirements. /// /// `alignment == 0` means no O_DIRECT (all I/O passes straight through). @@ -128,6 +139,92 @@ impl AlignedFile { abuf.write_to(&self.file)?; Ok(len) } + + /// Read into the buffers described by `iovecs`, starting at `offset`. + /// + /// # Safety + /// Every iovec must describe valid, writable memory of `iov_len` bytes. + pub(crate) unsafe fn read_vectored_at( + &self, + iovecs: &[libc::iovec], + offset: u64, + ) -> io::Result { + if iovecs.is_empty() { + return Ok(0); + } + if iovecs_are_aligned(self.alignment, iovecs, offset) { + // SAFETY: upheld by this fn contract. + let ret = unsafe { + libc::preadv( + self.file.as_raw_fd(), + iovecs.as_ptr(), + iovecs.len() as libc::c_int, + offset as libc::off_t, + ) + }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + return Ok(ret as usize); + } + let total_len = iovecs.iter().map(|iov| iov.iov_len).sum(); + self.read_unaligned(offset, total_len, |mut data| { + for iov in iovecs { + if data.is_empty() { + break; + } + let n = data.len().min(iov.iov_len); + // SAFETY: upheld by this fn contract. + let dst = unsafe { slice::from_raw_parts_mut(iov.iov_base as *mut u8, n) }; + dst.copy_from_slice(&data[..n]); + data = &data[n..]; + } + Ok(()) + }) + } + + /// Write the buffers described by `iovecs`, starting at `offset`. + /// + /// # Safety + /// Every iovec must describe valid, readable memory of `iov_len` bytes. + pub(crate) unsafe fn write_vectored_at( + &self, + iovecs: &[libc::iovec], + offset: u64, + ) -> io::Result { + if iovecs.is_empty() { + return Ok(0); + } + if iovecs_are_aligned(self.alignment, iovecs, offset) { + // SAFETY: upheld by this fn contract. + let ret = unsafe { + libc::pwritev( + self.file.as_raw_fd(), + iovecs.as_ptr(), + iovecs.len() as libc::c_int, + offset as libc::off_t, + ) + }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + return Ok(ret as usize); + } + let total_len = iovecs.iter().map(|iov| iov.iov_len).sum(); + self.write_unaligned(offset, total_len, |mut dst| { + for iov in iovecs { + if dst.is_empty() { + break; + } + let n = dst.len().min(iov.iov_len); + // SAFETY: upheld by this fn contract. + let src = unsafe { slice::from_raw_parts(iov.iov_base as *const u8, n) }; + dst[..n].copy_from_slice(src); + dst = &mut dst[n..]; + } + Ok(()) + }) + } } impl FileExt for AlignedFile { diff --git a/block/src/formats/raw/engine_sync.rs b/block/src/formats/raw/engine_sync.rs index 6678d2ecb..ba0a72aef 100644 --- a/block/src/formats/raw/engine_sync.rs +++ b/block/src/formats/raw/engine_sync.rs @@ -10,7 +10,6 @@ use std::os::unix::io::AsRawFd; use vmm_sys_util::eventfd::EventFd; -use super::{operation_is_aligned, run_unaligned_operation}; use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult}; use crate::sparse::{punch_hole, write_zeroes}; use crate::{AlignedFile, is_block_device}; @@ -46,52 +45,22 @@ impl AsyncIo for RawSync { self.alignment } - fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> { + fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> { let is_read = op.is_read(); + let iovecs = op.iovecs(); + let offset = op.offset() as u64; - let result = if operation_is_aligned(&op, self.alignment) { - let fd = self.raw_file.as_raw_fd(); - let offset = op.offset(); - let iovecs = op.iovecs(); - - let result = if is_read { - // SAFETY: the memory pointed to by `iovecs` is backed by the op, - // and valid for the kernel to write to by construction of - // AsyncIoOperation. - unsafe { - libc::preadv( - fd as libc::c_int, - iovecs.as_ptr(), - iovecs.len() as libc::c_int, - offset, - ) - } - } else { - // SAFETY: the memory pointed to by `iovecs` is backed by the op, - // and valid for the kernel to read from by construction of - // AsyncIoOperation. - unsafe { - libc::pwritev( - fd as libc::c_int, - iovecs.as_ptr(), - iovecs.len() as libc::c_int, - offset, - ) - } - }; - if result < 0 { - let error = io::Error::last_os_error(); - return Err(if is_read { - AsyncIoError::ReadVectored(error) - } else { - AsyncIoError::WriteVectored(error) - }); - } - result as i32 + let result = if is_read { + // SAFETY: op.iovecs() describes valid memory for iov_len bytes by + // construction of AsyncIoOperation. + unsafe { self.raw_file.read_vectored_at(iovecs, offset) } + .map_err(AsyncIoError::ReadVectored)? } else { - run_unaligned_operation(&self.raw_file, &mut op)? - }; - + // SAFETY: op.iovecs() describes valid memory for iov_len bytes by + // construction of AsyncIoOperation. + unsafe { self.raw_file.write_vectored_at(iovecs, offset) } + .map_err(AsyncIoError::WriteVectored)? + } as i32; self.completion_list .push_back(AsyncIoCompletion::from_operation(op, result)); self.eventfd.write(1).unwrap();