From 2fe775fce28d5d9e1acc447505c78e45f48a5721 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Tue, 12 May 2026 16:07:27 +0000 Subject: [PATCH] block: use BLKDISCARD/BLKZEROOUT ioctls for block devices Some block devices (ZFS volume) may require BLKDISCARD and BLKZEROOUT ioctls for discard and write_zeroes operations respectively. There is no good way to probe whether fallocate is supported on a block device. Arguably, punch_hole and write_zeroes are rare. Instead of having a complex scheme for the IO uring backend, we force it to always use ioctls. The code can be changed if the synchronized ioctls become a performance issue. Changes: - Detect block devices at construction time - Use BLKDISCARD ioctl for punch_hole (discard) on block devices - Use BLKZEROOUT ioctl for write_zeroes on block devices - Add BLKDISCARD/BLKZEROOUT to VirtioBlock seccomp whitelist - Keep fallocate() path for regular files (no behavior change) - Consolidate some helper functions to the new sparse module Signed-off-by: Wei Liu --- block/src/lib.rs | 2 + block/src/raw_async.rs | 43 ++++++++++- block/src/raw_async_aio.rs | 55 ++++---------- block/src/raw_sync.rs | 47 +++--------- block/src/sparse.rs | 104 ++++++++++++++++++++++++++ virtio-devices/src/seccomp_filters.rs | 13 +++- 6 files changed, 185 insertions(+), 79 deletions(-) create mode 100644 block/src/sparse.rs diff --git a/block/src/lib.rs b/block/src/lib.rs index bab8e74eb..dfeec593c 100644 --- a/block/src/lib.rs +++ b/block/src/lib.rs @@ -34,6 +34,8 @@ mod raw_async_io_tests; pub mod raw_disk; pub(crate) mod raw_sync; mod request; +mod sparse; +pub use sparse::{BLKDISCARD, BLKZEROOUT}; pub mod vhd; pub mod vhdx; pub mod vhdx_sync; diff --git a/block/src/raw_async.rs b/block/src/raw_async.rs index 79c84c05a..daf6775a9 100644 --- a/block/src/raw_async.rs +++ b/block/src/raw_async.rs @@ -11,13 +11,15 @@ use vmm_sys_util::eventfd::EventFd; use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult}; use crate::error::{BlockError, BlockErrorKind, BlockResult}; -use crate::{BatchRequest, RequestType, SECTOR_SIZE}; +use crate::sparse::{blkdiscard, blkzeroout}; +use crate::{BatchRequest, RequestType, SECTOR_SIZE, is_block_device}; pub struct RawFileAsync { fd: RawFd, io_uring: IoUring, eventfd: EventFd, alignment: u64, + is_block_device: bool, } impl RawFileAsync { @@ -34,13 +36,32 @@ impl RawFileAsync { .register_eventfd(eventfd.as_raw_fd()) .map_err(|e| BlockError::new(BlockErrorKind::Io, e))?; + let is_block_device = is_block_device(fd); + Ok(RawFileAsync { fd, io_uring, eventfd, alignment: SECTOR_SIZE, + is_block_device, }) } + + /// Queue an `IORING_OP_NOP` carrying `user_data` so a synchronously + /// completed operation (e.g. a BLK* ioctl) is reaped through the normal + /// io_uring completion path. + fn submit_nop(&mut self, user_data: u64) -> Result<(), Error> { + let (submitter, mut sq, _) = self.io_uring.split(); + // SAFETY: Nop carries no buffer; only `user_data` is consumed by the + // kernel. + unsafe { + sq.push(&opcode::Nop::new().build().user_data(user_data)) + .map_err(|e| Error::other(format!("Submission queue is full: {e:?}")))?; + }; + sq.sync(); + submitter.submit()?; + Ok(()) + } } impl AsyncIo for RawFileAsync { @@ -235,6 +256,18 @@ impl AsyncIo for RawFileAsync { } fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { + // Some block devices don't support fallocate(). Use ioctl instead. The assumption is that + // this happens rarely and we don't need to introduce unnecessary complexity by submitting + // a fallocate request, reaping ENOTSUPP in the completion routine, and reissuing the + // request with an ioctl. + if self.is_block_device { + blkdiscard(self.fd, offset, length).map_err(AsyncIoError::PunchHole)?; + // Deliver the completion through the normal io_uring path by + // queuing a NOP carrying `user_data`. The registered eventfd will + // fire when it completes, just like any other request. + return self.submit_nop(user_data).map_err(AsyncIoError::PunchHole); + } + let (submitter, mut sq, _) = self.io_uring.split(); let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; @@ -260,6 +293,14 @@ impl AsyncIo for RawFileAsync { } fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { + // Same rationale as punch_hole(). + if self.is_block_device { + blkzeroout(self.fd, offset, length).map_err(AsyncIoError::WriteZeroes)?; + return self + .submit_nop(user_data) + .map_err(AsyncIoError::WriteZeroes); + } + let (submitter, mut sq, _) = self.io_uring.split(); let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE; diff --git a/block/src/raw_async_aio.rs b/block/src/raw_async_aio.rs index 3636fd7fc..22e0896bc 100644 --- a/block/src/raw_async_aio.rs +++ b/block/src/raw_async_aio.rs @@ -8,13 +8,13 @@ use std::collections::VecDeque; use std::os::unix::io::{AsRawFd, RawFd}; -use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE}; use vmm_sys_util::aio; use vmm_sys_util::eventfd::EventFd; -use crate::SECTOR_SIZE; use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult}; use crate::error::{BlockError, BlockErrorKind, BlockResult}; +use crate::sparse::{punch_hole, write_zeroes}; +use crate::{SECTOR_SIZE, is_block_device}; pub struct RawFileAsyncAio { fd: RawFd, @@ -22,6 +22,7 @@ pub struct RawFileAsyncAio { eventfd: EventFd, alignment: u64, completion_list: VecDeque<(u64, i32)>, + is_block_device: bool, } impl RawFileAsyncAio { @@ -30,6 +31,7 @@ impl RawFileAsyncAio { EventFd::new(libc::EFD_NONBLOCK).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?; let ctx = aio::IoContext::new(queue_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?; + let is_block_device = is_block_device(fd); Ok(RawFileAsyncAio { fd, @@ -37,6 +39,7 @@ impl RawFileAsyncAio { eventfd, alignment: SECTOR_SIZE, completion_list: VecDeque::new(), + is_block_device, }) } } @@ -133,50 +136,22 @@ impl AsyncIo for RawFileAsyncAio { } fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - // Linux AIO has no IOCB command for fallocate, so perform the operation - // synchronously and signal completion via the completion list, matching - // the pattern used by the sync backend (RawFileSync). - let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; - - // SAFETY: FFI call with valid arguments - let result = unsafe { - libc::fallocate( - self.fd as libc::c_int, - mode, - offset as libc::off_t, - length as libc::off_t, - ) - }; - if result < 0 { - return Err(AsyncIoError::PunchHole(std::io::Error::last_os_error())); - } - - self.completion_list.push_back((user_data, result)); + // Linux AIO has no IOCB command for fallocate, so perform the + // operation synchronously and signal completion via the completion + // list, matching the pattern used by the sync backend (RawFileSync). + punch_hole(self.fd, self.is_block_device, offset, length) + .map_err(AsyncIoError::PunchHole)?; + self.completion_list.push_back((user_data, 0)); self.eventfd.write(1).unwrap(); Ok(()) } fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - // Linux AIO has no IOCB command for fallocate, so perform the operation - // synchronously and signal completion via the completion list, matching - // the pattern used by the sync backend (RawFileSync). - let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE; - - // SAFETY: FFI call with valid arguments - let result = unsafe { - libc::fallocate( - self.fd as libc::c_int, - mode, - offset as libc::off_t, - length as libc::off_t, - ) - }; - if result < 0 { - return Err(AsyncIoError::WriteZeroes(std::io::Error::last_os_error())); - } - - self.completion_list.push_back((user_data, result)); + // Same as punch_hole(). + write_zeroes(self.fd, self.is_block_device, offset, length) + .map_err(AsyncIoError::WriteZeroes)?; + self.completion_list.push_back((user_data, 0)); self.eventfd.write(1).unwrap(); Ok(()) diff --git a/block/src/raw_sync.rs b/block/src/raw_sync.rs index 659693f29..91ab4bac8 100644 --- a/block/src/raw_sync.rs +++ b/block/src/raw_sync.rs @@ -5,26 +5,29 @@ use std::collections::VecDeque; use std::os::unix::io::RawFd; -use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE}; use vmm_sys_util::eventfd::EventFd; -use crate::SECTOR_SIZE; use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult}; +use crate::sparse::{punch_hole, write_zeroes}; +use crate::{SECTOR_SIZE, is_block_device}; pub struct RawFileSync { fd: RawFd, eventfd: EventFd, completion_list: VecDeque<(u64, i32)>, alignment: u64, + is_block_device: bool, } impl RawFileSync { pub fn new(fd: RawFd) -> Self { + let is_block_device = is_block_device(fd); RawFileSync { fd, eventfd: EventFd::new(libc::EFD_NONBLOCK).expect("Failed creating EventFd for RawFile"), completion_list: VecDeque::new(), alignment: SECTOR_SIZE, + is_block_device, } } } @@ -108,46 +111,18 @@ impl AsyncIo for RawFileSync { } fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; - - // SAFETY: FFI call with valid arguments - let result = unsafe { - libc::fallocate( - self.fd as libc::c_int, - mode, - offset as libc::off_t, - length as libc::off_t, - ) - }; - if result < 0 { - return Err(AsyncIoError::PunchHole(std::io::Error::last_os_error())); - } - - self.completion_list.push_back((user_data, result)); + punch_hole(self.fd, self.is_block_device, offset, length) + .map_err(AsyncIoError::PunchHole)?; + self.completion_list.push_back((user_data, 0)); self.eventfd.write(1).unwrap(); - Ok(()) } fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> { - let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE; - - // SAFETY: FFI call with valid arguments - let result = unsafe { - libc::fallocate( - self.fd as libc::c_int, - mode, - offset as libc::off_t, - length as libc::off_t, - ) - }; - if result < 0 { - return Err(AsyncIoError::WriteZeroes(std::io::Error::last_os_error())); - } - - self.completion_list.push_back((user_data, result)); + write_zeroes(self.fd, self.is_block_device, offset, length) + .map_err(AsyncIoError::WriteZeroes)?; + self.completion_list.push_back((user_data, 0)); self.eventfd.write(1).unwrap(); - Ok(()) } } diff --git a/block/src/sparse.rs b/block/src/sparse.rs new file mode 100644 index 000000000..e23afb010 --- /dev/null +++ b/block/src/sparse.rs @@ -0,0 +1,104 @@ +// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause + +// Helpers for issuing `BLKDISCARD` / `BLKZEROOUT` ioctls on block devices, +// and the `punch_hole` / `write_zeroes` dispatchers used by the raw I/O +// backends. +// +// The kernel ioctl numbers and argument layout are stable userspace ABI +// (see `include/uapi/linux/fs.h`): +// +// ```c +// #define BLKDISCARD _IO(0x12, 119) /* arg: const __u64 range[2] = { start, len } */ +// #define BLKZEROOUT _IO(0x12, 127) /* arg: const __u64 range[2] = { start, len } */ +// ``` +// +// The kernel does `copy_from_user(range, arg, sizeof(range))`, i.e. it reads +// 16 bytes through the single pointer it is given, so we must pass a single +// `__u64[2]` array rather than two separate `*const u64` pointers. + +use std::io; +use std::os::unix::io::RawFd; + +use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE}; + +// `_IO(0x12, 119)` — issue a discard request to a block device. +pub const BLKDISCARD: libc::c_ulong = 0x1277; +// `_IO(0x12, 127)` — write zeroes to a range of a block device, with a +// kernel-side fallback to writing zero pages when the hardware has no native +// `WRITE_ZEROES`. +pub const BLKZEROOUT: libc::c_ulong = 0x127f; + +// Issue a `BLK*` range ioctl with proper `[start, len]` argument. +fn blk_range_ioctl(fd: RawFd, request: libc::c_ulong, offset: u64, length: u64) -> io::Result<()> { + let range: [u64; 2] = [offset, length]; + // SAFETY: `fd` is a valid block-device fd owned by the caller; `&range` + // is a 16-byte array matching the kernel's expected `__u64[2]` layout + // and lives for the duration of the call. + let ret = unsafe { libc::ioctl(fd, request as _, &range) }; + if ret == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +// Discard (TRIM/UNMAP) the byte range `[offset, offset + length)` on the +// block device referenced by `fd`. +pub(crate) fn blkdiscard(fd: RawFd, offset: u64, length: u64) -> io::Result<()> { + blk_range_ioctl(fd, BLKDISCARD, offset, length) +} + +// Zero the byte range `[offset, offset + length)` on the block device +// referenced by `fd`. The kernel falls back to writing explicit zero pages +// when the device has no hardware `WRITE_ZEROES`. +pub(crate) fn blkzeroout(fd: RawFd, offset: u64, length: u64) -> io::Result<()> { + blk_range_ioctl(fd, BLKZEROOUT, offset, length) +} + +// Punch a hole in `fd` over the byte range `[offset, offset + length)`. +// +// On block devices the kernel rejects `fallocate(PUNCH_HOLE)` (notably ZFS +// zvols), so route through `BLKDISCARD` instead. On regular files use +// `fallocate(FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)`. +pub(crate) fn punch_hole(fd: RawFd, is_blkdev: bool, offset: u64, length: u64) -> io::Result<()> { + if is_blkdev { + blkdiscard(fd, offset, length) + } else { + fallocate( + fd, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + offset, + length, + ) + } +} + +// Zero the byte range `[offset, offset + length)` in `fd`. +// +// Uses `BLKZEROOUT` on block devices (see [`punch_hole`] for the rationale) +// and `fallocate(FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE)` on regular +// files. +pub(crate) fn write_zeroes(fd: RawFd, is_blkdev: bool, offset: u64, length: u64) -> io::Result<()> { + if is_blkdev { + blkzeroout(fd, offset, length) + } else { + fallocate( + fd, + FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE, + offset, + length, + ) + } +} + +fn fallocate(fd: RawFd, mode: libc::c_int, offset: u64, length: u64) -> io::Result<()> { + // SAFETY: FFI call with a valid fd; fallocate touches no userspace memory. + let ret = unsafe { libc::fallocate(fd, mode, offset as libc::off_t, length as libc::off_t) }; + if ret == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} diff --git a/virtio-devices/src/seccomp_filters.rs b/virtio-devices/src/seccomp_filters.rs index 73c347ede..184122f1e 100644 --- a/virtio-devices/src/seccomp_filters.rs +++ b/virtio-devices/src/seccomp_filters.rs @@ -4,6 +4,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use block::{BLKDISCARD, BLKZEROOUT}; use libc::{FIONBIO, TIOCGWINSZ, TUNSETOFFLOAD}; use seccompiler::SeccompCmpOp::Eq; use seccompiler::{ @@ -113,8 +114,7 @@ fn virtio_block_thread_rules() -> Vec<(i64, Vec)> { (libc::SYS_fsync, vec![]), (libc::SYS_ftruncate, vec![]), (libc::SYS_getrandom, vec![]), - #[cfg(feature = "sev_snp")] - (libc::SYS_ioctl, create_mshv_sev_snp_ioctl_seccomp_rule()), + (libc::SYS_ioctl, create_virtio_block_ioctl_seccomp_rule()), (libc::SYS_io_destroy, vec![]), (libc::SYS_io_getevents, vec![]), (libc::SYS_io_submit, vec![]), @@ -131,6 +131,15 @@ fn virtio_block_thread_rules() -> Vec<(i64, Vec)> { ] } +fn create_virtio_block_ioctl_seccomp_rule() -> Vec { + or![ + and![Cond::new(1, ArgLen::Dword, Eq, BLKDISCARD as _).unwrap()], + and![Cond::new(1, ArgLen::Dword, Eq, BLKZEROOUT as _).unwrap()], + #[cfg(feature = "sev_snp")] + mshv_sev_snp_ioctl_seccomp_rule(), + ] +} + fn virtio_console_thread_rules() -> Vec<(i64, Vec)> { vec![ (libc::SYS_ioctl, create_virtio_console_ioctl_seccomp_rule()),