diff --git a/Cargo.lock b/Cargo.lock index 79bb26056..7f6bf16d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1637,6 +1637,7 @@ dependencies = [ "serde_json", "test_infra", "thiserror", + "vm-memory", "vmm-sys-util", ] diff --git a/performance-metrics/Cargo.toml b/performance-metrics/Cargo.toml index 776a46fc2..2d51d0e9a 100644 --- a/performance-metrics/Cargo.toml +++ b/performance-metrics/Cargo.toml @@ -13,6 +13,7 @@ serde = { workspace = true, features = ["derive", "rc"] } serde_json = { workspace = true } test_infra = { path = "../test_infra" } thiserror = { workspace = true } +vm-memory = { workspace = true, features = ["backend-mmap"] } vmm-sys-util = { workspace = true } [lints] diff --git a/performance-metrics/src/micro_bench_block.rs b/performance-metrics/src/micro_bench_block.rs index 4c67694ef..22d210df6 100644 --- a/performance-metrics/src/micro_bench_block.rs +++ b/performance-metrics/src/micro_bench_block.rs @@ -1,5 +1,7 @@ // Copyright 2026 The Cloud Hypervisor Authors. All rights reserved. // +// Copyright (c) Meta Platforms, Inc. and affiliates. +// // SPDX-License-Identifier: Apache-2.0 //! In process micro benchmarks for block layer internals. @@ -9,19 +11,18 @@ use std::time::Instant; +use block::async_io::AsyncIoOperation; use block::disk_file::AsyncDiskFile; use block::raw_disk::{RawBackend, RawDisk}; -use block::{BatchRequest, RequestType}; use crate::PerformanceTestControl; use crate::util::{ self, BLOCK_SIZE, L2_ENTRIES_PER_TABLE, QCOW_CLUSTER_SIZE, deterministic_permutation, - drain_async_completions, drain_completions, read_iovec, submit_reads, submit_writes, - write_iovec, + drain_async_completions, drain_completions, submit_reads, submit_writes, }; /// Submit num_ops AIO writes, wait for them all to land, then time -/// how long it takes to drain every completion via next_completed_request(). +/// how long it takes to drain every completion via next_completion(). /// /// Returns the drain wall clock time in seconds. pub fn micro_bench_aio_drain(control: &PerformanceTestControl) -> f64 { @@ -32,16 +33,14 @@ pub fn micro_bench_aio_drain(control: &PerformanceTestControl) -> f64 { .create_async_io(num_ops as u32) .expect("failed to create AIO context"); - let mut buf = vec![0xA5u8; BLOCK_SIZE as usize]; + let mem = util::guest_memory_buffer(BLOCK_SIZE as usize); + util::fill_guest_memory(&mem, BLOCK_SIZE as usize, 0xA5); // Submit all writes. for i in 0..num_ops { - let iovec = libc::iovec { - iov_base: buf.as_mut_ptr().cast(), - iov_len: buf.len(), - }; - aio.write_vectored((i as u64 * BLOCK_SIZE) as libc::off_t, &[iovec], i as u64) - .expect("write_vectored failed"); + let target = util::guest_memory_target(&mem, BLOCK_SIZE as usize); + aio.write_from_memory((i as u64 * BLOCK_SIZE) as libc::off_t, target, i as u64) + .expect("write_from_memory failed"); } // Wait until the eventfd signals that completions are available. @@ -51,7 +50,7 @@ pub fn micro_bench_aio_drain(control: &PerformanceTestControl) -> f64 { let start = Instant::now(); let mut drained = 0usize; while drained < num_ops { - if aio.next_completed_request().is_some() { + if aio.next_completion().is_some() { drained += 1; } } @@ -59,7 +58,7 @@ pub fn micro_bench_aio_drain(control: &PerformanceTestControl) -> f64 { } /// Read num_ops clusters from a prepopulated qcow2 image through the -/// QcowSync async_io path and time the total read_vectored wall clock. +/// QcowSync async_io path and time the total read wall clock. /// /// This exercises the hot read path: L2 lookup via map_clusters_for_read, /// pread64 for allocated data, and iovec scatter. @@ -69,12 +68,16 @@ pub fn micro_bench_qcow_read(control: &PerformanceTestControl) -> f64 { let num_ops = control.num_ops.expect("num_ops required") as usize; let (_tmp, disk) = util::qcow_tempfile(num_ops); let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); let elapsed = start.elapsed().as_secs_f64(); // Drain completions so Drop is clean. @@ -96,19 +99,18 @@ pub fn micro_bench_qcow_random_read(control: &PerformanceTestControl) -> f64 { let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); let indices = deterministic_permutation(num_ops); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); for (seq, &cluster_idx) in indices.iter().enumerate() { + let target = util::guest_memory_target(&mem, QCOW_CLUSTER_SIZE as usize); async_io - .read_vectored( + .read_to_memory( (cluster_idx as u64 * QCOW_CLUSTER_SIZE) as libc::off_t, - &[iovec], + target, seq as u64, ) - .expect("read_vectored failed"); + .expect("read_to_memory failed"); } let elapsed = start.elapsed().as_secs_f64(); @@ -118,7 +120,7 @@ pub fn micro_bench_qcow_random_read(control: &PerformanceTestControl) -> f64 { } /// Write num_ops clusters into an empty qcow2 image through the -/// QcowSync async_io path and time the total write_vectored wall clock. +/// QcowSync async_io path and time the total write wall clock. /// /// This exercises the write allocation path: map_cluster_for_write /// allocates a new cluster and bumps refcounts, then pwrite_all writes @@ -129,12 +131,17 @@ pub fn micro_bench_qcow_write(control: &PerformanceTestControl) -> f64 { let num_ops = control.num_ops.expect("num_ops required") as usize; let (_tmp, disk) = util::empty_qcow_tempfile(num_ops); let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); - - let buf = vec![0xA5u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = write_iovec(&buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); + util::fill_guest_memory(&mem, QCOW_CLUSTER_SIZE as usize, 0xA5); let start = Instant::now(); - submit_writes(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_writes( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); let elapsed = start.elapsed().as_secs_f64(); // Drain completions so Drop is clean. @@ -181,11 +188,17 @@ pub fn micro_bench_qcow_fsync(control: &PerformanceTestControl) -> f64 { let num_ops = control.num_ops.expect("num_ops required") as usize; let (_tmp, disk) = util::empty_qcow_tempfile(num_ops); let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); + util::fill_guest_memory(&mem, QCOW_CLUSTER_SIZE as usize, 0xA5); // Write num_ops clusters to dirty L2 and refcount metadata. - let buf = vec![0xA5u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = write_iovec(&buf); - submit_writes(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_writes( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); // Drain write completions. drain_completions(async_io.as_mut(), num_ops); @@ -211,12 +224,16 @@ pub fn micro_bench_qcow_backing_read(control: &PerformanceTestControl) -> f64 { let num_ops = control.num_ops.expect("num_ops required") as usize; let (_backing, _overlay, disk) = util::qcow_overlay_tempfile(num_ops); let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); let elapsed = start.elapsed().as_secs_f64(); drain_completions(async_io.as_mut(), num_ops); @@ -237,12 +254,17 @@ pub fn micro_bench_qcow_cow_write(control: &PerformanceTestControl) -> f64 { let num_ops = control.num_ops.expect("num_ops required") as usize; let (_backing, _overlay, disk) = util::qcow_overlay_tempfile(num_ops); let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); - - let buf = vec![0xBBu8; QCOW_CLUSTER_SIZE as usize]; - let iovec = write_iovec(&buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); + util::fill_guest_memory(&mem, QCOW_CLUSTER_SIZE as usize, 0xBB); let start = Instant::now(); - submit_writes(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_writes( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); let elapsed = start.elapsed().as_secs_f64(); drain_completions(async_io.as_mut(), num_ops); @@ -261,12 +283,16 @@ pub fn micro_bench_qcow_compressed_read(control: &PerformanceTestControl) -> f64 let num_ops = control.num_ops.expect("num_ops required") as usize; let (_tmp, disk) = util::compressed_qcow_tempfile(num_ops); let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); let elapsed = start.elapsed().as_secs_f64(); drain_completions(async_io.as_mut(), num_ops); @@ -276,7 +302,7 @@ pub fn micro_bench_qcow_compressed_read(control: &PerformanceTestControl) -> f64 /// Issue large multicluster reads from a prepopulated QCOW2 image. /// -/// Each read_vectored call spans `CLUSTERS_PER_READ` contiguous clusters +/// Each read call spans `CLUSTERS_PER_READ` contiguous clusters /// (8 x 64 KiB = 512 KiB). This exercises the mapping coalesce path /// where multiple L2 entries are merged into fewer host I/O operations. /// `num_ops` is the total number of clusters; reads are issued in @@ -291,12 +317,17 @@ pub fn micro_bench_qcow_multi_cluster_read(control: &PerformanceTestControl) -> let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); let read_size = CLUSTERS_PER_READ * QCOW_CLUSTER_SIZE as usize; - let mut buf = vec![0u8; read_size]; - let iovec = read_iovec(&mut buf); let num_reads = num_ops / CLUSTERS_PER_READ; + let mem = util::guest_memory_buffer(read_size); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_reads, read_size as u64, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_reads, + read_size as u64, + read_size, + ); let elapsed = start.elapsed().as_secs_f64(); drain_completions(async_io.as_mut(), num_reads); @@ -317,13 +348,17 @@ pub fn micro_bench_qcow_l2_cache_miss(control: &PerformanceTestControl) -> f64 { let num_ops = control.num_ops.expect("num_ops required") as usize; let (_tmp, disk) = util::sparse_qcow_tempfile(num_ops); let mut async_io = disk.create_async_io(1).expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let stride = L2_ENTRIES_PER_TABLE as u64 * QCOW_CLUSTER_SIZE; let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, stride, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + stride, + QCOW_CLUSTER_SIZE as usize, + ); let elapsed = start.elapsed().as_secs_f64(); drain_completions(async_io.as_mut(), num_ops); @@ -345,12 +380,16 @@ pub fn micro_bench_qcow_async_read(control: &PerformanceTestControl) -> f64 { let mut async_io = disk .create_async_io(num_ops as u32) .expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); // Drain all io_uring completions before stopping the clock. drain_async_completions(async_io.as_mut(), num_ops); @@ -368,29 +407,21 @@ pub fn micro_bench_qcow_batch_read(control: &PerformanceTestControl) -> f64 { let mut async_io = disk .create_async_io(num_ops as u32) .expect("create_async_io failed"); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); - let mut buf = vec![0u8; num_ops * QCOW_CLUSTER_SIZE as usize]; - - let batch: Vec = (0..num_ops) + let batch: Vec = (0..num_ops) .map(|i| { - let slice = - &mut buf[i * QCOW_CLUSTER_SIZE as usize..(i + 1) * QCOW_CLUSTER_SIZE as usize]; - BatchRequest { - offset: (i as u64 * QCOW_CLUSTER_SIZE) as libc::off_t, - iovecs: vec![libc::iovec { - iov_base: slice.as_mut_ptr().cast(), - iov_len: QCOW_CLUSTER_SIZE as usize, - }] - .into(), - user_data: i as u64, - request_type: RequestType::In, - } + AsyncIoOperation::read_to_memory( + (i as u64 * QCOW_CLUSTER_SIZE) as libc::off_t, + util::guest_memory_target(&mem, QCOW_CLUSTER_SIZE as usize), + i as u64, + ) }) .collect(); let start = Instant::now(); async_io - .submit_batch_requests(&batch) + .submit_batch_operations(batch) .expect("submit_batch_requests failed"); // Drain all io_uring completions before stopping the clock. @@ -410,19 +441,18 @@ pub fn micro_bench_qcow_async_random_read(control: &PerformanceTestControl) -> f .expect("create_async_io failed"); let indices = deterministic_permutation(num_ops); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); for (seq, &cluster_idx) in indices.iter().enumerate() { + let target = util::guest_memory_target(&mem, QCOW_CLUSTER_SIZE as usize); async_io - .read_vectored( + .read_to_memory( (cluster_idx as u64 * QCOW_CLUSTER_SIZE) as libc::off_t, - &[iovec], + target, seq as u64, ) - .expect("read_vectored failed"); + .expect("read_to_memory failed"); } drain_async_completions(async_io.as_mut(), num_ops); @@ -446,12 +476,17 @@ pub fn micro_bench_qcow_async_multi_cluster_read(control: &PerformanceTestContro .expect("create_async_io failed"); let read_size = CLUSTERS_PER_READ * QCOW_CLUSTER_SIZE as usize; - let mut buf = vec![0u8; read_size]; - let iovec = read_iovec(&mut buf); let num_reads = num_ops / CLUSTERS_PER_READ; + let mem = util::guest_memory_buffer(read_size); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_reads, read_size as u64, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_reads, + read_size as u64, + read_size, + ); drain_async_completions(async_io.as_mut(), num_reads); start.elapsed().as_secs_f64() @@ -470,12 +505,16 @@ pub fn micro_bench_qcow_async_backing_read(control: &PerformanceTestControl) -> let mut async_io = disk .create_async_io(num_ops as u32) .expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); drain_async_completions(async_io.as_mut(), num_ops); start.elapsed().as_secs_f64() @@ -492,12 +531,16 @@ pub fn micro_bench_qcow_async_compressed_read(control: &PerformanceTestControl) let mut async_io = disk .create_async_io(num_ops as u32) .expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); drain_async_completions(async_io.as_mut(), num_ops); start.elapsed().as_secs_f64() @@ -517,12 +560,17 @@ pub fn micro_bench_qcow_async_write(control: &PerformanceTestControl) -> f64 { let mut async_io = disk .create_async_io(num_ops as u32) .expect("create_async_io failed"); - - let buf = vec![0xA5u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = write_iovec(&buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); + util::fill_guest_memory(&mem, QCOW_CLUSTER_SIZE as usize, 0xA5); let start = Instant::now(); - submit_writes(async_io.as_mut(), num_ops, QCOW_CLUSTER_SIZE, &[iovec]); + submit_writes( + async_io.as_mut(), + &mem, + num_ops, + QCOW_CLUSTER_SIZE, + QCOW_CLUSTER_SIZE as usize, + ); drain_async_completions(async_io.as_mut(), num_ops); start.elapsed().as_secs_f64() @@ -538,13 +586,17 @@ pub fn micro_bench_qcow_async_l2_cache_miss(control: &PerformanceTestControl) -> let mut async_io = disk .create_async_io(num_ops as u32) .expect("create_async_io failed"); - - let mut buf = vec![0u8; QCOW_CLUSTER_SIZE as usize]; - let iovec = read_iovec(&mut buf); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); let stride = L2_ENTRIES_PER_TABLE as u64 * QCOW_CLUSTER_SIZE; let start = Instant::now(); - submit_reads(async_io.as_mut(), num_ops, stride, &[iovec]); + submit_reads( + async_io.as_mut(), + &mem, + num_ops, + stride, + QCOW_CLUSTER_SIZE as usize, + ); drain_async_completions(async_io.as_mut(), num_ops); start.elapsed().as_secs_f64() @@ -555,7 +607,7 @@ pub fn micro_bench_qcow_async_l2_cache_miss(control: &PerformanceTestControl) -> /// Builds a batch of num_ops write requests and submits them all at once /// through submit_batch_requests. Writes in QcowAsync are synchronous /// (COW path), so this measures whether batching reduces per-request -/// overhead compared to individual write_vectored calls. +/// overhead compared to individual write calls. /// /// Returns the total wall clock time in seconds. pub fn micro_bench_qcow_batch_write(control: &PerformanceTestControl) -> f64 { @@ -564,29 +616,22 @@ pub fn micro_bench_qcow_batch_write(control: &PerformanceTestControl) -> f64 { let mut async_io = disk .create_async_io(num_ops as u32) .expect("create_async_io failed"); + let mem = util::guest_memory_buffer(QCOW_CLUSTER_SIZE as usize); + util::fill_guest_memory(&mem, QCOW_CLUSTER_SIZE as usize, 0xA5); - let mut buf = vec![0xA5u8; num_ops * QCOW_CLUSTER_SIZE as usize]; - - let batch: Vec = (0..num_ops) + let batch: Vec = (0..num_ops) .map(|i| { - let slice = - &mut buf[i * QCOW_CLUSTER_SIZE as usize..(i + 1) * QCOW_CLUSTER_SIZE as usize]; - BatchRequest { - offset: (i as u64 * QCOW_CLUSTER_SIZE) as libc::off_t, - iovecs: vec![libc::iovec { - iov_base: slice.as_mut_ptr().cast(), - iov_len: QCOW_CLUSTER_SIZE as usize, - }] - .into(), - user_data: i as u64, - request_type: RequestType::Out, - } + AsyncIoOperation::write_from_memory( + (i as u64 * QCOW_CLUSTER_SIZE) as libc::off_t, + util::guest_memory_target(&mem, QCOW_CLUSTER_SIZE as usize), + i as u64, + ) }) .collect(); let start = Instant::now(); async_io - .submit_batch_requests(&batch) + .submit_batch_operations(batch) .expect("submit_batch_requests failed"); drain_async_completions(async_io.as_mut(), num_ops); diff --git a/performance-metrics/src/util.rs b/performance-metrics/src/util.rs index ce443c20b..6fb82e3dd 100644 --- a/performance-metrics/src/util.rs +++ b/performance-metrics/src/util.rs @@ -1,5 +1,7 @@ // Copyright 2026 The Cloud Hypervisor Authors. All rights reserved. // +// Copyright (c) Meta Platforms, Inc. and affiliates. +// // SPDX-License-Identifier: Apache-2.0 //! Shared benchmark helpers. @@ -8,12 +10,14 @@ use std::fs::File; use std::io::{ErrorKind, Seek, SeekFrom, Write}; use std::os::unix::fs::FileExt; use std::process::Command; +use std::sync::Arc; use std::thread; use std::time::Duration; -use block::async_io::AsyncIo; +use block::async_io::{AsyncIo, GuestMemoryTarget}; use block::qcow::{BackingFileConfig, ImageType, QcowFile, RawFile}; use block::qcow_disk::QcowDisk; +use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap}; use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::tempfile::TempFile; @@ -77,23 +81,7 @@ pub fn qcow_async_tempfile(num_clusters: usize) -> (TempFile, QcowDisk) { /// Drain `count` completions from a synchronous async_io backend. pub fn drain_completions(async_io: &mut dyn AsyncIo, count: usize) { for _ in 0..count { - async_io.next_completed_request(); - } -} - -/// Build an iovec suitable for a read into `buf`. -pub fn read_iovec(buf: &mut [u8]) -> libc::iovec { - libc::iovec { - iov_base: buf.as_mut_ptr().cast(), - iov_len: buf.len(), - } -} - -/// Build an iovec suitable for a write from `buf`. -pub fn write_iovec(buf: &[u8]) -> libc::iovec { - libc::iovec { - iov_base: buf.as_ptr().cast::().cast_mut(), - iov_len: buf.len(), + async_io.next_completion(); } } @@ -115,21 +103,81 @@ pub fn deterministic_permutation(n: usize) -> Vec { indices } -/// Submit `count` sequential read_vectored calls at `stride`-byte intervals. -pub fn submit_reads(async_io: &mut dyn AsyncIo, count: usize, stride: u64, iovec: &[libc::iovec]) { - for i in 0..count { - async_io - .read_vectored((i as u64 * stride) as libc::off_t, iovec, i as u64) - .expect("read_vectored failed"); +/// Create prefaulted guest memory for one reusable I/O range. +/// +/// The block microbenchmarks intentionally use this as a hot buffer to keep +/// cache behavior close to the borrowed-iovec benchmarks they replaced. +pub fn guest_memory_buffer(len: usize) -> Arc { + assert!( + len <= u32::MAX as usize, + "GuestMemoryTarget ranges are limited to u32 lengths" + ); + let total_len = len.max(1); + + let mem = Arc::new( + GuestMemoryMmap::from_ranges(&[(GuestAddress(0), total_len)]) + .expect("failed to create benchmark guest memory"), + ); + prefault_guest_memory(&mem, total_len); + mem +} + +fn prefault_guest_memory(mem: &Arc, total_len: usize) { + const PAGE_SIZE: usize = 4096; + + for offset in (0..total_len).step_by(PAGE_SIZE) { + mem.write_slice(&[0], GuestAddress(offset as u64)) + .expect("failed to prefault benchmark guest memory"); } } -/// Submit `count` sequential write_vectored calls at `stride`-byte intervals. -pub fn submit_writes(async_io: &mut dyn AsyncIo, count: usize, stride: u64, iovec: &[libc::iovec]) { +/// Create a target for the reusable benchmark guest-memory range. +pub fn guest_memory_target(mem: &Arc, len: usize) -> GuestMemoryTarget { + assert!( + len <= u32::MAX as usize, + "GuestMemoryTarget ranges are limited to u32 lengths" + ); + let range = [(GuestAddress(0), len as u32)]; + + GuestMemoryTarget::new(Arc::clone(mem), &range).expect("failed to create guest memory target") +} + +/// Fill the reusable benchmark guest-memory range with one byte pattern. +pub fn fill_guest_memory(mem: &Arc, len: usize, value: u8) { + let buf = vec![value; len]; + mem.write_slice(&buf, GuestAddress(0)) + .expect("failed to initialize benchmark guest memory"); +} + +/// Submit `count` sequential read calls at `stride`-byte intervals. +pub fn submit_reads( + async_io: &mut dyn AsyncIo, + mem: &Arc, + count: usize, + stride: u64, + len: usize, +) { for i in 0..count { + let target = guest_memory_target(mem, len); async_io - .write_vectored((i as u64 * stride) as libc::off_t, iovec, i as u64) - .expect("write_vectored failed"); + .read_to_memory((i as u64 * stride) as libc::off_t, target, i as u64) + .expect("read_to_memory failed"); + } +} + +/// Submit `count` sequential write calls at `stride`-byte intervals. +pub fn submit_writes( + async_io: &mut dyn AsyncIo, + mem: &Arc, + count: usize, + stride: u64, + len: usize, +) { + for i in 0..count { + let target = guest_memory_target(mem, len); + async_io + .write_from_memory((i as u64 * stride) as libc::off_t, target, i as u64) + .expect("write_from_memory failed"); } } @@ -139,7 +187,7 @@ pub fn drain_async_completions(async_io: &mut dyn AsyncIo, count: usize) { let mut drained = 0usize; while drained < count { wait_for_eventfd(async_io.notifier()); - while async_io.next_completed_request().is_some() { + while async_io.next_completion().is_some() { drained += 1; } }