diff --git a/block/src/formats/raw/engine_aio.rs b/block/src/formats/raw/engine_aio.rs index ba7385f27..49a527769 100644 --- a/block/src/formats/raw/engine_aio.rs +++ b/block/src/formats/raw/engine_aio.rs @@ -94,13 +94,8 @@ impl AsyncIo for RawAio { // Linux AIO has no IOCB command for fallocate, so perform the // operation synchronously and signal completion via the completion // list, matching the pattern used by the sync backend (RawSync). - punch_hole( - self.raw_file.as_raw_fd(), - self.is_block_device, - offset, - length, - ) - .map_err(AsyncIoError::PunchHole)?; + punch_hole(&mut self.raw_file, self.is_block_device, offset, length) + .map_err(AsyncIoError::PunchHole)?; self.data_io .inject_completion(AsyncIoCompletion::new(user_data, 0, None)); @@ -109,13 +104,8 @@ impl AsyncIo for RawAio { fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { // Same as punch_hole(). - write_zeroes( - self.raw_file.as_raw_fd(), - self.is_block_device, - offset, - length, - ) - .map_err(AsyncIoError::WriteZeroes)?; + write_zeroes(&mut self.raw_file, self.is_block_device, offset, length) + .map_err(AsyncIoError::WriteZeroes)?; self.data_io .inject_completion(AsyncIoCompletion::new(user_data, 0, None)); diff --git a/block/src/formats/raw/engine_sync.rs b/block/src/formats/raw/engine_sync.rs index 44480813d..6678d2ecb 100644 --- a/block/src/formats/raw/engine_sync.rs +++ b/block/src/formats/raw/engine_sync.rs @@ -120,13 +120,8 @@ impl AsyncIo for RawSync { } fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - punch_hole( - self.raw_file.as_raw_fd(), - self.is_block_device, - offset, - length, - ) - .map_err(AsyncIoError::PunchHole)?; + punch_hole(&mut self.raw_file, self.is_block_device, offset, length) + .map_err(AsyncIoError::PunchHole)?; self.completion_list .push_back(AsyncIoCompletion::new(user_data, 0, None)); self.eventfd.write(1).unwrap(); @@ -134,13 +129,8 @@ impl AsyncIo for RawSync { } fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - write_zeroes( - self.raw_file.as_raw_fd(), - self.is_block_device, - offset, - length, - ) - .map_err(AsyncIoError::WriteZeroes)?; + write_zeroes(&mut self.raw_file, self.is_block_device, offset, length) + .map_err(AsyncIoError::WriteZeroes)?; self.completion_list .push_back(AsyncIoCompletion::new(user_data, 0, None)); self.eventfd.write(1).unwrap(); diff --git a/block/src/formats/raw/engine_uring.rs b/block/src/formats/raw/engine_uring.rs index 68a58530f..0ef2c21bd 100644 --- a/block/src/formats/raw/engine_uring.rs +++ b/block/src/formats/raw/engine_uring.rs @@ -6,7 +6,6 @@ use std::os::unix::io::AsRawFd; -use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE}; use vmm_sys_util::eventfd::EventFd; use super::{operation_is_aligned, run_unaligned_operation}; @@ -14,7 +13,7 @@ use crate::async_io::{ AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult, UringDataIo, }; use crate::error::{BlockError, BlockErrorKind, BlockResult}; -use crate::sparse::{blkdiscard, blkzeroout}; +use crate::sparse::{punch_hole, write_zeroes}; use crate::{AlignedFile, is_block_device}; pub(crate) struct RawAsync { @@ -119,44 +118,27 @@ impl AsyncIo for RawAsync { } fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - // Some block devices don't support fallocate(). Use ioctl instead. The assumption is that - // this happens rarely and we don't need to introduce unnecessary complexity by submitting - // a fallocate request, reaping ENOTSUPP in the completion routine, and reissuing the - // request with an ioctl. - if self.is_block_device { - blkdiscard(self.raw_file.as_raw_fd(), offset, length) - .map_err(AsyncIoError::PunchHole)?; - // Deliver the completion through the normal io_uring path by - // queuing a NOP carrying `user_data`. The registered eventfd will - // fire when it completes, just like any other request. - return self - .data_io - .submit_nop(user_data) - .map_err(AsyncIoError::PunchHole); - } - - let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; - + // Run synchronously rather than submitting a fallocate request through + // the ring. This avoids reaping ENOTSUPP in the completion routine and + // reissuing the request, and lets the sparse helper handle the ioctl + // path for block devices and the write fallback for unsupported + // filesystems. + punch_hole(&mut self.raw_file, self.is_block_device, offset, length) + .map_err(AsyncIoError::PunchHole)?; + // Deliver the completion through the normal io_uring path by + // queuing a NOP carrying `user_data`. The registered eventfd will + // fire when it completes, just like any other request. self.data_io - .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data) + .submit_nop(user_data) .map_err(AsyncIoError::PunchHole) } fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { // Same rationale as punch_hole(). - if self.is_block_device { - blkzeroout(self.raw_file.as_raw_fd(), offset, length) - .map_err(AsyncIoError::WriteZeroes)?; - return self - .data_io - .submit_nop(user_data) - .map_err(AsyncIoError::WriteZeroes); - } - - let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE; - + write_zeroes(&mut self.raw_file, self.is_block_device, offset, length) + .map_err(AsyncIoError::WriteZeroes)?; self.data_io - .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data) + .submit_nop(user_data) .map_err(AsyncIoError::WriteZeroes) } } diff --git a/block/src/io/async_io/uring_data_io.rs b/block/src/io/async_io/uring_data_io.rs index 12167983e..c0920af54 100644 --- a/block/src/io/async_io/uring_data_io.rs +++ b/block/src/io/async_io/uring_data_io.rs @@ -211,25 +211,6 @@ impl UringDataIo { ) } - /// Submits a fallocate operation carrying `user_data`. - pub fn submit_fallocate( - &mut self, - fd: RawFd, - offset: u64, - length: u64, - mode: i32, - user_data: u64, - ) -> io::Result<()> { - self.submit_kernel_entry( - user_data, - &opcode::Fallocate::new(types::Fd(fd), length) - .offset(offset) - .mode(mode) - .build() - .user_data(user_data), - ) - } - /// Injects a completion that did not come from a kernel CQE. /// /// The notifier is signaled so callers can drain it with @@ -360,13 +341,6 @@ mod tests { data_io.submit_nop(7).unwrap_err().kind(), io::ErrorKind::AlreadyExists ); - assert_eq!( - data_io - .submit_fallocate(fd, 0, 512, 0, 7) - .unwrap_err() - .kind(), - io::ErrorKind::AlreadyExists - ); let completion = wait_for_completion(&mut data_io); assert_eq!(completion.user_data, 7); diff --git a/block/src/sparse.rs b/block/src/sparse.rs index e23afb010..55787fcc6 100644 --- a/block/src/sparse.rs +++ b/block/src/sparse.rs @@ -19,9 +19,11 @@ // `__u64[2]` array rather than two separate `*const u64` pointers. use std::io; -use std::os::unix::io::RawFd; +use std::os::unix::io::{AsRawFd, RawFd}; -use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE}; +use vmm_sys_util::write_zeroes::{PunchHole, WriteZeroesAt}; + +use crate::AlignedFile; // `_IO(0x12, 119)` — issue a discard request to a block device. pub const BLKDISCARD: libc::c_ulong = 0x1277; @@ -61,44 +63,45 @@ pub(crate) fn blkzeroout(fd: RawFd, offset: u64, length: u64) -> io::Result<()> // // On block devices the kernel rejects `fallocate(PUNCH_HOLE)` (notably ZFS // zvols), so route through `BLKDISCARD` instead. On regular files use -// `fallocate(FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)`. -pub(crate) fn punch_hole(fd: RawFd, is_blkdev: bool, offset: u64, length: u64) -> io::Result<()> { +// `PunchHole`, falling back to `WriteZeroesAt` on EOPNOTSUPP (e.g. tmpfs). +pub(crate) fn punch_hole( + file: &mut AlignedFile, + is_blkdev: bool, + offset: u64, + length: u64, +) -> io::Result<()> { if is_blkdev { - blkdiscard(fd, offset, length) - } else { - fallocate( - fd, - FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - offset, - length, - ) + return match blkdiscard(file.as_raw_fd(), offset, length) { + Ok(()) => Ok(()), + Err(e) if e.raw_os_error() == Some(libc::EOPNOTSUPP) => Ok(()), + Err(e) => Err(e), + }; + } + match file.punch_hole(offset, length) { + Ok(()) => Ok(()), + Err(e) if e.raw_os_error() == Some(libc::EOPNOTSUPP) => { + file.write_all_zeroes_at(offset, length as usize)?; + Ok(()) + } + Err(e) => Err(e), } } // Zero the byte range `[offset, offset + length)` in `fd`. // // Uses `BLKZEROOUT` on block devices (see [`punch_hole`] for the rationale) -// and `fallocate(FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE)` on regular -// files. -pub(crate) fn write_zeroes(fd: RawFd, is_blkdev: bool, offset: u64, length: u64) -> io::Result<()> { +// and `WriteZeroesAt` on regular files, which tries fallocate and falls +// back to positional writes on EOPNOTSUPP (e.g. tmpfs). +pub(crate) fn write_zeroes( + file: &mut AlignedFile, + is_blkdev: bool, + offset: u64, + length: u64, +) -> io::Result<()> { if is_blkdev { - blkzeroout(fd, offset, length) - } else { - fallocate( - fd, - FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE, - offset, - length, - ) + return blkzeroout(file.as_raw_fd(), offset, length); } + file.write_all_zeroes_at(offset, length as usize)?; + Ok(()) } -fn fallocate(fd: RawFd, mode: libc::c_int, offset: u64, length: u64) -> io::Result<()> { - // SAFETY: FFI call with a valid fd; fallocate touches no userspace memory. - let ret = unsafe { libc::fallocate(fd, mode, offset as libc::off_t, length as libc::off_t) }; - if ret == 0 { - Ok(()) - } else { - Err(io::Error::last_os_error()) - } -}