diff --git a/block/src/aligned_file.rs b/block/src/aligned_file.rs index 1dd919147..67a96efed 100644 --- a/block/src/aligned_file.rs +++ b/block/src/aligned_file.rs @@ -3,10 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::fs::{File, Metadata}; -use std::io; use std::os::fd::{AsFd, BorrowedFd}; use std::os::unix::fs::FileExt; use std::os::unix::io::{AsRawFd, RawFd}; +use std::{io, slice}; use vmm_sys_util::file_traits::FileSync; use vmm_sys_util::seek_hole::SeekHole; @@ -24,6 +24,17 @@ fn is_aligned(alignment: usize, buf_ptr: usize, len: usize, offset: u64) -> bool && offset.is_multiple_of(alignment as u64)) } +/// True when `offset` and every iovec base/length satisfy `alignment` +/// (`alignment == 0` means no O_DIRECT, so everything is "aligned"). +fn iovecs_are_aligned(alignment: usize, iovecs: &[libc::iovec], offset: u64) -> bool { + alignment == 0 + || (offset.is_multiple_of(alignment as u64) + && iovecs.iter().all(|iov| { + (iov.iov_base as usize).is_multiple_of(alignment) + && iov.iov_len.is_multiple_of(alignment) + })) +} + /// A `File` that transparently satisfies O_DIRECT alignment requirements. /// /// `alignment == 0` means no O_DIRECT (all I/O passes straight through). @@ -128,6 +139,92 @@ impl AlignedFile { abuf.write_to(&self.file)?; Ok(len) } + + /// Read into the buffers described by `iovecs`, starting at `offset`. + /// + /// # Safety + /// Every iovec must describe valid, writable memory of `iov_len` bytes. + pub(crate) unsafe fn read_vectored_at( + &self, + iovecs: &[libc::iovec], + offset: u64, + ) -> io::Result { + if iovecs.is_empty() { + return Ok(0); + } + if iovecs_are_aligned(self.alignment, iovecs, offset) { + // SAFETY: upheld by this fn contract. + let ret = unsafe { + libc::preadv( + self.file.as_raw_fd(), + iovecs.as_ptr(), + iovecs.len() as libc::c_int, + offset as libc::off_t, + ) + }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + return Ok(ret as usize); + } + let total_len = iovecs.iter().map(|iov| iov.iov_len).sum(); + self.read_unaligned(offset, total_len, |mut data| { + for iov in iovecs { + if data.is_empty() { + break; + } + let n = data.len().min(iov.iov_len); + // SAFETY: upheld by this fn contract. + let dst = unsafe { slice::from_raw_parts_mut(iov.iov_base as *mut u8, n) }; + dst.copy_from_slice(&data[..n]); + data = &data[n..]; + } + Ok(()) + }) + } + + /// Write the buffers described by `iovecs`, starting at `offset`. + /// + /// # Safety + /// Every iovec must describe valid, readable memory of `iov_len` bytes. + pub(crate) unsafe fn write_vectored_at( + &self, + iovecs: &[libc::iovec], + offset: u64, + ) -> io::Result { + if iovecs.is_empty() { + return Ok(0); + } + if iovecs_are_aligned(self.alignment, iovecs, offset) { + // SAFETY: upheld by this fn contract. + let ret = unsafe { + libc::pwritev( + self.file.as_raw_fd(), + iovecs.as_ptr(), + iovecs.len() as libc::c_int, + offset as libc::off_t, + ) + }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + return Ok(ret as usize); + } + let total_len = iovecs.iter().map(|iov| iov.iov_len).sum(); + self.write_unaligned(offset, total_len, |mut dst| { + for iov in iovecs { + if dst.is_empty() { + break; + } + let n = dst.len().min(iov.iov_len); + // SAFETY: upheld by this fn contract. + let src = unsafe { slice::from_raw_parts(iov.iov_base as *const u8, n) }; + dst[..n].copy_from_slice(src); + dst = &mut dst[n..]; + } + Ok(()) + }) + } } impl FileExt for AlignedFile { diff --git a/block/src/formats/raw/engine_sync.rs b/block/src/formats/raw/engine_sync.rs index 6678d2ecb..ba0a72aef 100644 --- a/block/src/formats/raw/engine_sync.rs +++ b/block/src/formats/raw/engine_sync.rs @@ -10,7 +10,6 @@ use std::os::unix::io::AsRawFd; use vmm_sys_util::eventfd::EventFd; -use super::{operation_is_aligned, run_unaligned_operation}; use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult}; use crate::sparse::{punch_hole, write_zeroes}; use crate::{AlignedFile, is_block_device}; @@ -46,52 +45,22 @@ impl AsyncIo for RawSync { self.alignment } - fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> { + fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> { let is_read = op.is_read(); + let iovecs = op.iovecs(); + let offset = op.offset() as u64; - let result = if operation_is_aligned(&op, self.alignment) { - let fd = self.raw_file.as_raw_fd(); - let offset = op.offset(); - let iovecs = op.iovecs(); - - let result = if is_read { - // SAFETY: the memory pointed to by `iovecs` is backed by the op, - // and valid for the kernel to write to by construction of - // AsyncIoOperation. - unsafe { - libc::preadv( - fd as libc::c_int, - iovecs.as_ptr(), - iovecs.len() as libc::c_int, - offset, - ) - } - } else { - // SAFETY: the memory pointed to by `iovecs` is backed by the op, - // and valid for the kernel to read from by construction of - // AsyncIoOperation. - unsafe { - libc::pwritev( - fd as libc::c_int, - iovecs.as_ptr(), - iovecs.len() as libc::c_int, - offset, - ) - } - }; - if result < 0 { - let error = io::Error::last_os_error(); - return Err(if is_read { - AsyncIoError::ReadVectored(error) - } else { - AsyncIoError::WriteVectored(error) - }); - } - result as i32 + let result = if is_read { + // SAFETY: op.iovecs() describes valid memory for iov_len bytes by + // construction of AsyncIoOperation. + unsafe { self.raw_file.read_vectored_at(iovecs, offset) } + .map_err(AsyncIoError::ReadVectored)? } else { - run_unaligned_operation(&self.raw_file, &mut op)? - }; - + // SAFETY: op.iovecs() describes valid memory for iov_len bytes by + // construction of AsyncIoOperation. + unsafe { self.raw_file.write_vectored_at(iovecs, offset) } + .map_err(AsyncIoError::WriteVectored)? + } as i32; self.completion_list .push_back(AsyncIoCompletion::from_operation(op, result)); self.eventfd.write(1).unwrap();