kata-containers/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2026-08-05 02:19:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

vm-virtio, virtio-devices: Split device implementation from virt queues

Browse Source 
Split the generic virtio code (queues and device type) from the
VirtioDevice trait, transport and device implementations.

This also simplifies the feature handling in vhost_user_backend as the
vm-virtio crate is no longer has any features.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>
This commit is contained in:
Rob Bradford
2020-07-02 13:25:19 +01:00
parent 9a628edfcf
commit 2a6eb31d5b
51 changed files with 301 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1357
vm-virtio/src/block.rs
View File

File diff suppressed because it is too large Load Diff

677
vm-virtio/src/console.rs
View File

@@ -1,677 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
use super::Error as DeviceError;
use super::{
ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
VirtioInterruptType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
};
use crate::VirtioInterrupt;
use anyhow::anyhow;
use libc::EFD_NONBLOCK;
use serde::ser::{Serialize, SerializeStruct, Serializer};
use std::cmp;
use std::collections::VecDeque;
use std::fs::File;
use std::io;
use std::io::Write;
use std::ops::DerefMut;
use std::os::unix::io::{AsRawFd, FromRawFd};
use std::result;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use vm_memory::{ByteValued, Bytes, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
use vm_migration::{
Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
Transportable,
};
use vmm_sys_util::eventfd::EventFd;
const QUEUE_SIZE: u16 = 256;
const NUM_QUEUES: usize = 2;
const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE; NUM_QUEUES];
// New descriptors are pending on the virtio queue.
const INPUT_QUEUE_EVENT: DeviceEventT = 0;
const OUTPUT_QUEUE_EVENT: DeviceEventT = 1;
// Some input from the VMM is ready to be injected into the VM.
const INPUT_EVENT: DeviceEventT = 2;
// The device has been dropped.
const KILL_EVENT: DeviceEventT = 3;
// Console configuration change event is triggered.
const CONFIG_EVENT: DeviceEventT = 4;
// The device should be paused.
const PAUSE_EVENT: DeviceEventT = 5;
//Console size feature bit
const VIRTIO_CONSOLE_F_SIZE: u64 = 0;
#[derive(Copy, Clone, Debug, Default, Deserialize)]
#[repr(C, packed)]
pub struct VirtioConsoleConfig {
cols: u16,
rows: u16,
max_nr_ports: u32,
emerg_wr: u32,
}
// We must explicitly implement Serialize since the structure is packed and
// it's unsafe to borrow from a packed structure. And by default, if we derive
// Serialize from serde, it will borrow the values from the structure.
// That's why this implementation copies each field separately before it
// serializes the entire structure field by field.
impl Serialize for VirtioConsoleConfig {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let cols = self.cols;
let rows = self.rows;
let max_nr_ports = self.max_nr_ports;
let emerg_wr = self.emerg_wr;
let mut virtio_console_config = serializer.serialize_struct("VirtioConsoleConfig", 12)?;
virtio_console_config.serialize_field("cols", &cols)?;
virtio_console_config.serialize_field("rows", &rows)?;
virtio_console_config.serialize_field("max_nr_ports", &max_nr_ports)?;
virtio_console_config.serialize_field("emerg_wr", &emerg_wr)?;
virtio_console_config.end()
}
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioConsoleConfig {}
struct ConsoleEpollHandler {
queues: Vec<Queue>,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
in_buffer: Arc<Mutex<VecDeque<u8>>>,
out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
input_queue_evt: EventFd,
output_queue_evt: EventFd,
input_evt: EventFd,
config_evt: EventFd,
kill_evt: EventFd,
pause_evt: EventFd,
}
impl ConsoleEpollHandler {
/*
* Each port of virtio console device has one receive
* queue. One or more empty buffers are placed by the
* dirver in the receive queue for incoming data. Here,
* we place the input data to these empty buffers.
*/
fn process_input_queue(&mut self) -> bool {
let mut in_buffer = self.in_buffer.lock().unwrap();
let recv_queue = &mut self.queues[0]; //receiveq
let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
let mut used_count = 0;
if in_buffer.is_empty() {
return false;
}
let mem = self.mem.memory();
for avail_desc in recv_queue.iter(&mem) {
let len = cmp::min(avail_desc.len as u32, in_buffer.len() as u32);
let source_slice = in_buffer.drain(..len as usize).collect::<Vec<u8>>();
if let Err(e) = mem.write_slice(&source_slice[..], avail_desc.addr) {
error!("Failed to write slice: {:?}", e);
recv_queue.go_to_previous_position();
break;
}
used_desc_heads[used_count] = (avail_desc.index, len);
used_count += 1;
if in_buffer.is_empty() {
break;
}
}
for &(desc_index, len) in &used_desc_heads[..used_count] {
recv_queue.add_used(&mem, desc_index, len);
}
used_count > 0
}
/*
* Each port of virtio console device has one transmit
* queue. For outgoing data, characters are placed in
* the transmit queue by the driver. Therefore, here
* we read data from the transmit queue and flush them
* to the referenced address.
*/
fn process_output_queue(&mut self) -> bool {
let trans_queue = &mut self.queues[1]; //transmitq
let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
let mut used_count = 0;
let mem = self.mem.memory();
for avail_desc in trans_queue.iter(&mem) {
let len;
let mut out = self.out.lock().unwrap();
let _ = mem.write_to(
avail_desc.addr,
&mut out.deref_mut(),
avail_desc.len as usize,
);
let _ = out.flush();
len = avail_desc.len;
used_desc_heads[used_count] = (avail_desc.index, len);
used_count += 1;
}
for &(desc_index, len) in &used_desc_heads[..used_count] {
trans_queue.add_used(&mem, desc_index, len);
}
used_count > 0
}
fn signal_used_queue(&self) -> result::Result<(), DeviceError> {
self.interrupt_cb
.trigger(&VirtioInterruptType::Queue, Some(&self.queues[0]))
.map_err(|e| {
error!("Failed to signal used queue: {:?}", e);
DeviceError::FailedSignalingUsedQueue(e)
})
}
fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
// Create the epoll file descriptor
let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
// Use 'File' to enforce closing on 'epoll_fd'
let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
// Add events
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.input_queue_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(INPUT_QUEUE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.output_queue_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(OUTPUT_QUEUE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.input_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(INPUT_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.config_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(CONFIG_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.kill_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.pause_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
const EPOLL_EVENTS_LEN: usize = 100;
let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
// Before jumping into the epoll loop, check if the device is expected
// to be in a paused state. This is helpful for the restore code path
// as the device thread should not start processing anything before the
// device has been resumed.
while paused.load(Ordering::SeqCst) {
thread::park();
}
'epoll: loop {
let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
Ok(res) => res,
Err(e) => {
if e.kind() == io::ErrorKind::Interrupted {
// It's well defined from the epoll_wait() syscall
// documentation that the epoll loop can be interrupted
// before any of the requested events occurred or the
// timeout expired. In both those cases, epoll_wait()
// returns an error of type EINTR, but this should not
// be considered as a regular error. Instead it is more
// appropriate to retry, by calling into epoll_wait().
continue;
}
return Err(DeviceError::EpollWait(e));
}
};
for event in events.iter().take(num_events) {
let ev_type = event.data as u16;
match ev_type {
INPUT_QUEUE_EVENT => {
if let Err(e) = self.input_queue_evt.read() {
error!("Failed to get queue event: {:?}", e);
break 'epoll;
} else if self.process_input_queue() {
if let Err(e) = self.signal_used_queue() {
error!("Failed to signal used queue: {:?}", e);
break 'epoll;
}
}
}
OUTPUT_QUEUE_EVENT => {
if let Err(e) = self.output_queue_evt.read() {
error!("Failed to get queue event: {:?}", e);
break 'epoll;
} else {
self.process_output_queue();
}
}
INPUT_EVENT => {
if let Err(e) = self.input_evt.read() {
error!("Failed to get input event: {:?}", e);
break 'epoll;
} else if self.process_input_queue() {
if let Err(e) = self.signal_used_queue() {
error!("Failed to signal used queue: {:?}", e);
break 'epoll;
}
}
}
CONFIG_EVENT => {
if let Err(e) = self.config_evt.read() {
error!("Failed to get config event: {:?}", e);
break 'epoll;
} else if let Err(e) = self
.interrupt_cb
.trigger(&VirtioInterruptType::Config, None)
{
error!("Failed to signal console driver: {:?}", e);
}
}
KILL_EVENT => {
debug!("KILL_EVENT received, stopping epoll loop");
break 'epoll;
}
PAUSE_EVENT => {
debug!("PAUSE_EVENT received, pausing virtio-console epoll loop");
// We loop here to handle spurious park() returns.
// Until we have not resumed, the paused boolean will
// be true.
while paused.load(Ordering::SeqCst) {
thread::park();
}
// Drain pause event after the device has been resumed.
// This ensures the pause event has been seen by each
// and every thread related to this virtio device.
let _ = self.pause_evt.read();
}
_ => {
error!("Unknown event for virtio-console");
}
}
}
}
Ok(())
}
}
/// Input device.
pub struct ConsoleInput {
input_evt: EventFd,
config_evt: EventFd,
in_buffer: Arc<Mutex<VecDeque<u8>>>,
config: Arc<Mutex<VirtioConsoleConfig>>,
acked_features: AtomicU64,
}
impl ConsoleInput {
pub fn queue_input_bytes(&self, input: &[u8]) {
let mut in_buffer = self.in_buffer.lock().unwrap();
in_buffer.extend(input);
let _ = self.input_evt.write(1);
}
pub fn update_console_size(&self, cols: u16, rows: u16) {
if self
.acked_features
.fetch_and(1u64 << VIRTIO_CONSOLE_F_SIZE, Ordering::SeqCst)
!= 0
{
self.config.lock().unwrap().update_console_size(cols, rows);
//Send the interrupt to the driver
let _ = self.config_evt.write(1);
}
}
}
impl VirtioConsoleConfig {
pub fn new(cols: u16, rows: u16) -> Self {
VirtioConsoleConfig {
cols,
rows,
max_nr_ports: 1u32,
emerg_wr: 0u32,
}
}
pub fn update_console_size(&mut self, cols: u16, rows: u16) {
self.cols = cols;
self.rows = rows;
}
}
/// Virtio device for exposing console to the guest OS through virtio.
pub struct Console {
id: String,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
avail_features: u64,
acked_features: u64,
config: Arc<Mutex<VirtioConsoleConfig>>,
input: Arc<ConsoleInput>,
out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
paused: Arc<AtomicBool>,
}
#[derive(Serialize, Deserialize)]
pub struct ConsoleState {
avail_features: u64,
acked_features: u64,
config: VirtioConsoleConfig,
in_buffer: VecDeque<u8>,
}
impl Console {
/// Create a new virtio console device that gets random data from /dev/urandom.
pub fn new(
id: String,
out: Box<dyn io::Write + Send + Sync + 'static>,
cols: u16,
rows: u16,
iommu: bool,
) -> io::Result<(Console, Arc<ConsoleInput>)> {
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_CONSOLE_F_SIZE;
if iommu {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
let input_evt = EventFd::new(EFD_NONBLOCK).unwrap();
let config_evt = EventFd::new(EFD_NONBLOCK).unwrap();
let console_config = Arc::new(Mutex::new(VirtioConsoleConfig::new(cols, rows)));
let console_input = Arc::new(ConsoleInput {
input_evt,
config_evt,
in_buffer: Arc::new(Mutex::new(VecDeque::new())),
config: console_config.clone(),
acked_features: AtomicU64::new(0),
});
Ok((
Console {
id,
kill_evt: None,
pause_evt: None,
avail_features,
acked_features: 0u64,
config: console_config,
input: console_input.clone(),
out: Arc::new(Mutex::new(out)),
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
paused: Arc::new(AtomicBool::new(false)),
},
console_input,
))
}
fn state(&self) -> ConsoleState {
ConsoleState {
avail_features: self.avail_features,
acked_features: self.acked_features,
config: *(self.config.lock().unwrap()),
in_buffer: self.input.in_buffer.lock().unwrap().clone(),
}
}
fn set_state(&mut self, state: &ConsoleState) -> io::Result<()> {
self.avail_features = state.avail_features;
self.acked_features = state.acked_features;
*(self.config.lock().unwrap()) = state.config;
*(self.input.in_buffer.lock().unwrap()) = state.in_buffer.clone();
Ok(())
}
}
impl Drop for Console {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
}
}
impl VirtioDevice for Console {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_CONSOLE as u32
}
fn queue_max_sizes(&self) -> &[u16] {
QUEUE_SIZES
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("Received acknowledge request for unknown feature.");
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, offset: u64, mut data: &mut [u8]) {
let config = self.config.lock().unwrap();
let config_slice = config.as_slice();
let config_len = config_slice.len() as u64;
if offset >= config_len {
error!("Failed to read config space");
return;
}
if let Some(end) = offset.checked_add(data.len() as u64) {
// This write can't fail, offset and end are checked against config_len.
data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
.unwrap();
}
}
fn write_config(&mut self, _offset: u64, _data: &[u8]) {
warn!("No device specific configration requires write");
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
queues: Vec<Queue>,
mut queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
NUM_QUEUES,
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
// Save the interrupt EventFD as we need to return it on reset
// but clone it to pass into the thread.
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
self.input
.acked_features
.store(self.acked_features, Ordering::Relaxed);
if (self.acked_features & (1u64 << VIRTIO_CONSOLE_F_SIZE)) != 0 {
if let Err(e) = interrupt_cb.trigger(&VirtioInterruptType::Config, None) {
error!("Failed to signal console driver: {:?}", e);
}
}
let mut handler = ConsoleEpollHandler {
queues,
mem,
interrupt_cb,
in_buffer: self.input.in_buffer.clone(),
out: self.out.clone(),
input_queue_evt: queue_evts.remove(0),
output_queue_evt: queue_evts.remove(0),
input_evt: self.input.input_evt.try_clone().unwrap(),
config_evt: self.input.config_evt.try_clone().unwrap(),
kill_evt,
pause_evt,
};
let paused = self.paused.clone();
let mut epoll_threads = Vec::new();
thread::Builder::new()
.name("virtio_console".to_string())
.spawn(move || handler.run(paused))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone the virtio-console epoll thread: {}", e);
ActivateError::BadActivate
})?;
self.epoll_threads = Some(epoll_threads);
Ok(())
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
}
virtio_pausable!(Console);
impl Snapshottable for Console {
fn id(&self) -> String {
self.id.clone()
}
fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
let snapshot =
serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
let mut console_snapshot = Snapshot::new(self.id.as_str());
console_snapshot.add_data_section(SnapshotDataSection {
id: format!("{}-section", self.id),
snapshot,
});
Ok(console_snapshot)
}
fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
if let Some(console_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
let console_state = match serde_json::from_slice(&console_section.snapshot) {
Ok(state) => state,
Err(error) => {
return Err(MigratableError::Restore(anyhow!(
"Could not deserialize CONSOLE {}",
error
)))
}
};
return self.set_state(&console_state).map_err(|e| {
MigratableError::Restore(anyhow!("Could not restore CONSOLE state {:?}", e))
});
}
Err(MigratableError::Restore(anyhow!(
"Could not find CONSOLE snapshot section"
)))
}
}
impl Transportable for Console {}
impl Migratable for Console {}

274
vm-virtio/src/device.rs
View File

@@ -1,274 +0,0 @@
// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-BSD-3-Clause file.
//
// Copyright © 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
use crate::{ActivateResult, Error, Queue};
use std::collections::HashMap;
use std::num::Wrapping;
use std::sync::Arc;
use vm_memory::{GuestAddress, GuestMemoryAtomic, GuestMemoryMmap, GuestUsize};
use vmm_sys_util::eventfd::EventFd;
pub enum VirtioInterruptType {
Config,
Queue,
}
pub trait VirtioInterrupt: Send + Sync {
fn trigger(
&self,
int_type: &VirtioInterruptType,
queue: Option<&Queue>,
) -> std::result::Result<(), std::io::Error>;
fn notifier(
&self,
_int_type: &VirtioInterruptType,
_queue: Option<&Queue>,
) -> Option<&EventFd> {
None
}
}
pub type VirtioIommuRemapping =
Box<dyn Fn(u64) -> std::result::Result<u64, std::io::Error> + Send + Sync>;
#[derive(Clone)]
pub struct UserspaceMapping {
pub host_addr: u64,
pub mem_slot: u32,
pub addr: GuestAddress,
pub len: GuestUsize,
pub mergeable: bool,
}
#[derive(Clone)]
pub struct VirtioSharedMemory {
pub offset: u64,
pub len: u64,
}
#[derive(Clone)]
pub struct VirtioSharedMemoryList {
pub host_addr: u64,
pub mem_slot: u32,
pub addr: GuestAddress,
pub len: GuestUsize,
pub region_list: Vec<VirtioSharedMemory>,
}
/// Trait for virtio devices to be driven by a virtio transport.
///
/// The lifecycle of a virtio device is to be moved to a virtio transport, which will then query the
/// device. Once the guest driver has configured the device, `VirtioDevice::activate` will be called
/// and all the events, memory, and queues for device operation will be moved into the device.
/// Optionally, a virtio device can implement device reset in which it returns said resources and
/// resets its internal.
pub trait VirtioDevice: Send {
/// The virtio device type.
fn device_type(&self) -> u32;
/// The maximum size of each queue that this device supports.
fn queue_max_sizes(&self) -> &[u16];
/// The set of feature bits that this device supports.
fn features(&self) -> u64 {
0
}
/// Acknowledges that this set of features should be enabled.
fn ack_features(&mut self, value: u64) {
let _ = value;
}
/// Reads this device configuration space at `offset`.
fn read_config(&self, offset: u64, data: &mut [u8]);
/// Writes to this device configuration space at `offset`.
fn write_config(&mut self, offset: u64, data: &[u8]);
/// Activates this device for real usage.
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_evt: Arc<dyn VirtioInterrupt>,
queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
) -> ActivateResult;
/// Optionally deactivates this device and returns ownership of the guest memory map, interrupt
/// event, and queue events.
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
None
}
/// Returns the list of shared memory regions required by the device.
fn get_shm_regions(&self) -> Option<VirtioSharedMemoryList> {
None
}
/// Updates the list of shared memory regions required by the device.
fn set_shm_regions(
&mut self,
_shm_regions: VirtioSharedMemoryList,
) -> std::result::Result<(), Error> {
std::unimplemented!()
}
fn iommu_translate(&self, addr: u64) -> u64 {
addr
}
/// Some devices may need to do some explicit shutdown work. This method
/// may be implemented to do this. The VMM should call shutdown() on
/// every device as part of shutting down the VM. Acting on the device
/// after a shutdown() can lead to unpredictable results.
fn shutdown(&mut self) {}
fn update_memory(&mut self, _mem: &GuestMemoryMmap) -> std::result::Result<(), Error> {
Ok(())
}
/// Returns the list of userspace mappings associated with this device.
fn userspace_mappings(&self) -> Vec<UserspaceMapping> {
Vec::new()
}
/// Return the counters that this device exposes
fn counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>> {
None
}
}
/// Trait providing address translation the same way a physical DMA remapping
/// table would provide translation between an IOVA and a physical address.
/// The goal of this trait is to be used by virtio devices to perform the
/// address translation before they try to read from the guest physical address.
/// On the other side, the implementation itself should be provided by the code
/// emulating the IOMMU for the guest.
pub trait DmaRemapping: Send + Sync {
fn translate(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>;
}
#[macro_export]
macro_rules! virtio_pausable_trait_definition {
() => {
trait VirtioPausable {
fn virtio_pause(&mut self) -> std::result::Result<(), MigratableError>;
fn virtio_resume(&mut self) -> std::result::Result<(), MigratableError>;
}
};
}
#[macro_export]
macro_rules! virtio_pausable_trait_inner {
() => {
// This is the common Pausable trait implementation for virtio.
fn virtio_pause(&mut self) -> result::Result<(), MigratableError> {
debug!(
"Pausing virtio-{}",
VirtioDeviceType::from(self.device_type())
);
self.paused.store(true, Ordering::SeqCst);
if let Some(pause_evt) = &self.pause_evt {
pause_evt
.write(1)
.map_err(|e| MigratableError::Pause(e.into()))?;
}
Ok(())
}
fn virtio_resume(&mut self) -> result::Result<(), MigratableError> {
debug!(
"Resuming virtio-{}",
VirtioDeviceType::from(self.device_type())
);
self.paused.store(false, Ordering::SeqCst);
if let Some(epoll_threads) = &self.epoll_threads {
for i in 0..epoll_threads.len() {
epoll_threads[i].thread().unpark();
}
}
Ok(())
}
};
}
#[macro_export]
macro_rules! virtio_pausable_trait {
($type:ident) => {
virtio_pausable_trait_definition!();
impl VirtioPausable for $type {
virtio_pausable_trait_inner!();
}
};
($type:ident, T: $($bounds:tt)+) => {
virtio_pausable_trait_definition!();
impl<T: $($bounds)+ > VirtioPausable for $type<T> {
virtio_pausable_trait_inner!();
}
};
}
#[macro_export]
macro_rules! virtio_pausable_inner {
($type:ident) => {
fn pause(&mut self) -> result::Result<(), MigratableError> {
self.virtio_pause()
}
fn resume(&mut self) -> result::Result<(), MigratableError> {
self.virtio_resume()
}
};
}
#[macro_export]
macro_rules! virtio_pausable {
($type:ident) => {
virtio_pausable_trait!($type);
impl Pausable for $type {
virtio_pausable_inner!($type);
}
};
// For type bound virtio types
($type:ident, T: $($bounds:tt)+) => {
virtio_pausable_trait!($type, T: $($bounds)+);
impl<T: $($bounds)+ > Pausable for $type<T> {
virtio_pausable_inner!($type);
}
};
}
#[macro_export]
macro_rules! virtio_ctrl_q_pausable {
($type:ident) => {
virtio_pausable_trait!($type);
impl Pausable for $type {
fn pause(&mut self) -> result::Result<(), MigratableError> {
self.virtio_pause()
}
fn resume(&mut self) -> result::Result<(), MigratableError> {
self.virtio_resume()?;
if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread {
ctrl_queue_epoll_thread.thread().unpark();
}
Ok(())
}
}
};
}

1128
vm-virtio/src/iommu.rs
View File

File diff suppressed because it is too large Load Diff

110
vm-virtio/src/lib.rs
View File

@@ -8,64 +8,20 @@
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
//! Implements virtio devices, queues, and transport mechanisms.
//! Implements virtio queues
extern crate arc_swap;
extern crate epoll;
#[macro_use]
extern crate log;
#[cfg(feature = "pci_support")]
extern crate pci;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate serde_json;
extern crate vhost_rs;
extern crate virtio_bindings;
extern crate vm_device;
extern crate vm_memory;
use std::fmt;
use std::io;
#[macro_use]
mod device;
pub mod block;
mod console;
mod iommu;
pub mod mem;
pub mod net;
pub mod net_util;
mod pmem;
pub mod queue;
mod rng;
pub mod vsock;
pub use queue::*;
pub mod transport;
pub mod vhost_user;
pub use self::block::*;
pub use self::console::*;
pub use self::device::*;
pub use self::iommu::*;
pub use self::mem::*;
pub use self::net::*;
pub use self::net_util::*;
pub use self::pmem::*;
pub use self::queue::*;
pub use self::rng::*;
pub use self::vsock::*;
const DEVICE_INIT: u32 = 0x00;
const DEVICE_ACKNOWLEDGE: u32 = 0x01;
const DEVICE_DRIVER: u32 = 0x02;
const DEVICE_DRIVER_OK: u32 = 0x04;
const DEVICE_FEATURES_OK: u32 = 0x08;
const DEVICE_FAILED: u32 = 0x80;
const VIRTIO_F_VERSION_1: u32 = 32;
const VIRTIO_F_IOMMU_PLATFORM: u32 = 33;
const VIRTIO_F_IN_ORDER: u32 = 35;
pub type VirtioIommuRemapping =
Box<dyn Fn(u64) -> std::result::Result<u64, std::io::Error> + Send + Sync>;
// Types taken from linux/virtio_ids.h
#[derive(Copy, Clone, Debug)]
@@ -134,61 +90,3 @@ impl fmt::Display for VirtioDeviceType {
write!(f, "{}", output)
}
}
#[allow(dead_code)]
const INTERRUPT_STATUS_USED_RING: u32 = 0x1;
#[allow(dead_code)]
const INTERRUPT_STATUS_CONFIG_CHANGED: u32 = 0x2;
#[cfg(feature = "pci_support")]
const VIRTIO_MSI_NO_VECTOR: u16 = 0xffff;
#[derive(Debug)]
pub enum ActivateError {
EpollCtl(std::io::Error),
BadActivate,
/// Queue number is not correct
BadQueueNum,
/// Failed to clone Kill event
CloneKillEventFd,
/// Failed to create Vhost-user interrupt eventfd
VhostIrqCreate,
/// Failed to setup vhost-user daemon.
VhostUserSetup(vhost_user::Error),
/// Failed to setup vhost-user daemon.
VhostUserNetSetup(vhost_user::Error),
/// Failed to setup vhost-user daemon.
VhostUserBlkSetup(vhost_user::Error),
/// Failed to reset vhost-user daemon.
VhostUserReset(vhost_user::Error),
}
pub type ActivateResult = std::result::Result<(), ActivateError>;
pub type DeviceEventT = u16;
#[derive(Debug)]
pub enum Error {
FailedReadingQueue {
event_type: &'static str,
underlying: io::Error,
},
FailedReadTap,
FailedSignalingUsedQueue(io::Error),
PayloadExpected,
UnknownEvent {
device: &'static str,
event: DeviceEventT,
},
IoError(io::Error),
RegisterListener(io::Error),
UnregisterListener(io::Error),
EpollCreateFd(io::Error),
EpollCtl(io::Error),
EpollWait(io::Error),
FailedSignalingDriver(io::Error),
VhostUserUpdateMemory(vhost_user::Error),
EventfdError(io::Error),
SetShmRegionsNotSupported,
EpollHander(String),
NoMemoryConfigured,
}

996
vm-virtio/src/mem.rs
View File

@@ -1,996 +0,0 @@
// Copyright (c) 2020 Ant Financial
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::Error as DeviceError;
use super::{
ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
VirtioDeviceType, VIRTIO_F_VERSION_1,
};
use crate::{VirtioInterrupt, VirtioInterruptType};
use libc::EFD_NONBLOCK;
use std::cmp;
use std::fs::File;
use std::io::{self, Write};
use std::mem::size_of;
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::result;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc;
use std::sync::{Arc, Mutex};
use std::thread;
use vm_memory::{
Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
GuestMemoryError, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap,
};
use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
use vmm_sys_util::eventfd::EventFd;
const QUEUE_SIZE: u16 = 128;
const NUM_QUEUES: usize = 1;
const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE];
// Use 2 MiB alignment so transparent hugepages can be used by KVM.
pub const VIRTIO_MEM_DEFAULT_BLOCK_SIZE: u64 = 512 * 4096;
const VIRTIO_MEM_USABLE_EXTENT: u64 = 256 * 1024 * 1024;
// Request processed successfully, applicable for
// - VIRTIO_MEM_REQ_PLUG
// - VIRTIO_MEM_REQ_UNPLUG
// - VIRTIO_MEM_REQ_UNPLUG_ALL
// - VIRTIO_MEM_REQ_STATE
const VIRTIO_MEM_RESP_ACK: u16 = 0;
// Request denied - e.g. trying to plug more than requested, applicable for
// - VIRTIO_MEM_REQ_PLUG
const VIRTIO_MEM_RESP_NACK: u16 = 1;
// Request cannot be processed right now, try again later, applicable for
// - VIRTIO_MEM_REQ_PLUG
// - VIRTIO_MEM_REQ_UNPLUG
// - VIRTIO_MEM_REQ_UNPLUG_ALL
// VIRTIO_MEM_RESP_BUSY: u16 = 2;
// Error in request (e.g. addresses/alignemnt), applicable for
// - VIRTIO_MEM_REQ_PLUG
// - VIRTIO_MEM_REQ_UNPLUG
// - VIRTIO_MEM_REQ_STATE
const VIRTIO_MEM_RESP_ERROR: u16 = 3;
// State of memory blocks is "plugged"
const VIRTIO_MEM_STATE_PLUGGED: u16 = 0;
// State of memory blocks is "unplugged"
const VIRTIO_MEM_STATE_UNPLUGGED: u16 = 1;
// State of memory blocks is "mixed"
const VIRTIO_MEM_STATE_MIXED: u16 = 2;
// request to plug memory blocks
const VIRTIO_MEM_REQ_PLUG: u16 = 0;
// request to unplug memory blocks
const VIRTIO_MEM_REQ_UNPLUG: u16 = 1;
// request to unplug all blocks and shrink the usable size
const VIRTIO_MEM_REQ_UNPLUG_ALL: u16 = 2;
// request information about the plugged state of memory blocks
const VIRTIO_MEM_REQ_STATE: u16 = 3;
// Get resize event.
const RESIZE_EVENT: DeviceEventT = 0;
// New descriptors are pending on the virtio queue.
const QUEUE_AVAIL_EVENT: DeviceEventT = 1;
// The device has been dropped.
const KILL_EVENT: DeviceEventT = 2;
// The device should be paused.
const PAUSE_EVENT: DeviceEventT = 3;
#[derive(Debug)]
pub enum Error {
// Guest gave us bad memory addresses.
GuestMemory(GuestMemoryError),
// Guest gave us a write only descriptor that protocol says to read from.
UnexpectedWriteOnlyDescriptor,
// Guest gave us a read only descriptor that protocol says to write to.
UnexpectedReadOnlyDescriptor,
// Guest gave us too few descriptors in a descriptor chain.
DescriptorChainTooShort,
// Guest gave us a buffer that was too short to use.
BufferLengthTooSmall,
// Guest sent us invalid request.
InvalidRequest,
// Failed to EventFd write.
EventFdWriteFail(std::io::Error),
// Failed to EventFd try_clone.
EventFdTryCloneFail(std::io::Error),
// Failed to MpscRecv.
MpscRecvFail(mpsc::RecvError),
// Resize invalid argument
ResizeInval(String),
// Fail to resize trigger
ResizeTriggerFail(DeviceError),
}
// Got from qemu/include/standard-headers/linux/virtio_mem.h
// rust union doesn't support std::default::Default that
// need by mem.read_obj.
// Then move virtio_mem_req_plug, virtio_mem_req_unplug and
// virtio_mem_req_state to virtio_mem_req.
#[repr(C)]
#[derive(Copy, Clone, Debug, Default)]
struct VirtioMemReq {
req_type: u16,
padding: [u16; 3],
addr: u64,
nb_blocks: u16,
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioMemReq {}
// Got from qemu/include/standard-headers/linux/virtio_mem.h
#[repr(C)]
#[derive(Copy, Clone, Debug, Default)]
struct VirtioMemRespState {
state: u16,
}
#[repr(C)]
#[derive(Copy, Clone, Debug, Default)]
struct VirtioMemResp {
resp_type: u16,
padding: [u16; 3],
state: VirtioMemRespState,
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioMemResp {}
// Got from qemu/include/standard-headers/linux/virtio_mem.h
#[repr(C, packed)]
#[derive(Copy, Clone, Debug, Default)]
struct VirtioMemConfig {
// Block size and alignment. Cannot change.
block_size: u32,
// Valid with VIRTIO_MEM_F_ACPI_PXM. Cannot change.
node_id: u16,
padding: u16,
// Start address of the memory region. Cannot change.
addr: u64,
// Region size (maximum). Cannot change.
region_size: u64,
// Currently usable region size. Can grow up to region_size. Can
// shrink due to VIRTIO_MEM_REQ_UNPLUG_ALL (in which case no config
// update will be sent).
usable_region_size: u64,
// Currently used size. Changes due to plug/unplug requests, but no
// config updates will be sent.
plugged_size: u64,
// Requested size. New plug requests cannot exceed it. Can change.
requested_size: u64,
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioMemConfig {}
struct Request {
req: VirtioMemReq,
status_addr: GuestAddress,
}
impl Request {
fn parse(
avail_desc: &DescriptorChain,
mem: &GuestMemoryMmap,
) -> result::Result<Request, Error> {
// The head contains the request type which MUST be readable.
if avail_desc.is_write_only() {
return Err(Error::UnexpectedWriteOnlyDescriptor);
}
if avail_desc.len as usize != size_of::<VirtioMemReq>() {
return Err(Error::InvalidRequest);
}
let req: VirtioMemReq = mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?;
let status_desc = avail_desc
.next_descriptor()
.ok_or(Error::DescriptorChainTooShort)?;
// The status MUST always be writable
if !status_desc.is_write_only() {
return Err(Error::UnexpectedReadOnlyDescriptor);
}
if (status_desc.len as usize) < size_of::<VirtioMemResp>() {
return Err(Error::BufferLengthTooSmall);
}
Ok(Request {
req,
status_addr: status_desc.addr,
})
}
}
pub struct Resize {
size: Arc<AtomicU64>,
tx: mpsc::Sender<Result<(), Error>>,
rx: Option<mpsc::Receiver<Result<(), Error>>>,
evt: EventFd,
}
impl Resize {
pub fn new() -> io::Result<Self> {
let (tx, rx) = mpsc::channel();
Ok(Resize {
size: Arc::new(AtomicU64::new(0)),
tx,
rx: Some(rx),
evt: EventFd::new(EFD_NONBLOCK)?,
})
}
pub fn try_clone(&self) -> Result<Self, Error> {
Ok(Resize {
size: self.size.clone(),
tx: self.tx.clone(),
rx: None,
evt: self.evt.try_clone().map_err(Error::EventFdTryCloneFail)?,
})
}
pub fn work(&self, size: u64) -> Result<(), Error> {
if let Some(rx) = &self.rx {
self.size.store(size, Ordering::SeqCst);
self.evt.write(1).map_err(Error::EventFdWriteFail)?;
rx.recv().map_err(Error::MpscRecvFail)?
} else {
panic!("work should not work with cloned resize")
}
}
fn get_size(&self) -> u64 {
self.size.load(Ordering::SeqCst)
}
fn send(&self, r: Result<(), Error>) -> Result<(), mpsc::SendError<Result<(), Error>>> {
self.tx.send(r)
}
}
struct MemEpollHandler {
host_addr: u64,
host_fd: Option<RawFd>,
mem_state: Vec<bool>,
config: Arc<Mutex<VirtioMemConfig>>,
resize: Resize,
queue: Queue,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
queue_evt: EventFd,
kill_evt: EventFd,
pause_evt: EventFd,
}
struct StateChangeRequest<'a> {
config: VirtioMemConfig,
addr: u64,
size: u64,
nb_blocks: u16,
mem_state: &'a mut Vec<bool>,
host_addr: u64,
host_fd: Option<RawFd>,
plug: bool,
}
impl MemEpollHandler {
fn virtio_mem_valid_range(config: &VirtioMemConfig, addr: u64, size: u64) -> bool {
// address properly aligned?
if addr % config.block_size as u64 != 0 {
return false;
}
// reasonable size
if addr + size <= addr || size == 0 {
return false;
}
// start address in usable range?
if addr < config.addr || addr >= config.addr + config.usable_region_size {
return false;
}
// end address in usable range?
if addr + size > config.addr + config.usable_region_size {
return false;
}
true
}
fn virtio_mem_check_bitmap(
bit_index: usize,
nb_blocks: u16,
mem_state: &[bool],
plug: bool,
) -> bool {
for state in mem_state.iter().skip(bit_index).take(nb_blocks as usize) {
if *state != plug {
return false;
}
}
true
}
fn virtio_mem_set_bitmap(
bit_index: usize,
nb_blocks: u16,
mem_state: &mut Vec<bool>,
plug: bool,
) {
for state in mem_state
.iter_mut()
.skip(bit_index)
.take(nb_blocks as usize)
{
*state = plug;
}
}
fn virtio_mem_state_change_request(r: StateChangeRequest) -> u16 {
if r.plug && (r.config.plugged_size + r.size > r.config.requested_size) {
return VIRTIO_MEM_RESP_NACK;
}
if !MemEpollHandler::virtio_mem_valid_range(&r.config, r.addr, r.size) {
return VIRTIO_MEM_RESP_ERROR;
}
let offset = r.addr - r.config.addr;
let bit_index = (offset / r.config.block_size as u64) as usize;
if !MemEpollHandler::virtio_mem_check_bitmap(bit_index, r.nb_blocks, r.mem_state, !r.plug) {
return VIRTIO_MEM_RESP_ERROR;
}
if !r.plug {
if let Some(fd) = r.host_fd {
let res = unsafe {
libc::fallocate64(
fd,
libc::FALLOC_FL_PUNCH_HOLE | libc::FALLOC_FL_KEEP_SIZE,
offset as libc::off64_t,
r.size as libc::off64_t,
)
};
if res != 0 {
error!("fallocate64 get error {}", io::Error::last_os_error());
return VIRTIO_MEM_RESP_ERROR;
}
}
let res = unsafe {
libc::madvise(
(r.host_addr + offset) as *mut libc::c_void,
r.size as libc::size_t,
libc::MADV_DONTNEED,
)
};
if res != 0 {
error!("madvise get error {}", io::Error::last_os_error());
return VIRTIO_MEM_RESP_ERROR;
}
}
MemEpollHandler::virtio_mem_set_bitmap(bit_index, r.nb_blocks, r.mem_state, r.plug);
VIRTIO_MEM_RESP_ACK
}
fn virtio_mem_unplug_all(
config: VirtioMemConfig,
mem_state: &mut Vec<bool>,
host_addr: u64,
host_fd: Option<RawFd>,
) -> u16 {
for x in 0..(config.region_size / config.block_size as u64) as usize {
if mem_state[x] {
let resp_type =
MemEpollHandler::virtio_mem_state_change_request(StateChangeRequest {
config,
addr: config.addr + x as u64 * config.block_size as u64,
size: config.block_size as u64,
nb_blocks: 1,
mem_state,
host_addr,
host_fd,
plug: false,
});
if resp_type != VIRTIO_MEM_RESP_ACK {
return resp_type;
}
mem_state[x] = false;
}
}
VIRTIO_MEM_RESP_ACK
}
fn virtio_mem_state_request(
config: VirtioMemConfig,
addr: u64,
nb_blocks: u16,
mem_state: &mut Vec<bool>,
) -> (u16, u16) {
let size: u64 = nb_blocks as u64 * config.block_size as u64;
let resp_type = if MemEpollHandler::virtio_mem_valid_range(&config, addr, size) {
VIRTIO_MEM_RESP_ACK
} else {
VIRTIO_MEM_RESP_ERROR
};
let offset = addr - config.addr;
let bit_index = (offset / config.block_size as u64) as usize;
let resp_state =
if MemEpollHandler::virtio_mem_check_bitmap(bit_index, nb_blocks, mem_state, true) {
VIRTIO_MEM_STATE_PLUGGED
} else if MemEpollHandler::virtio_mem_check_bitmap(
bit_index, nb_blocks, mem_state, false,
) {
VIRTIO_MEM_STATE_UNPLUGGED
} else {
VIRTIO_MEM_STATE_MIXED
};
(resp_type, resp_state)
}
fn virtio_mem_send_response(
mem: &GuestMemoryMmap,
resp_type: u16,
resp_state: u16,
status_addr: GuestAddress,
) -> u32 {
let mut resp = VirtioMemResp::default();
resp.resp_type = resp_type;
resp.state.state = resp_state;
match mem.write_obj(resp, status_addr) {
Ok(_) => size_of::<VirtioMemResp>() as u32,
Err(e) => {
error!("bad guest memory address: {}", e);
0
}
}
}
fn signal(&self, int_type: &VirtioInterruptType) -> result::Result<(), DeviceError> {
self.interrupt_cb
.trigger(int_type, Some(&self.queue))
.map_err(|e| {
error!("Failed to signal used queue: {:?}", e);
DeviceError::FailedSignalingUsedQueue(e)
})
}
fn process_queue(&mut self) -> bool {
let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
let mut used_count = 0;
let mem = self.mem.memory();
for avail_desc in self.queue.iter(&mem) {
let len = match Request::parse(&avail_desc, &mem) {
Err(e) => {
error!("failed parse VirtioMemReq: {:?}", e);
0
}
Ok(r) => {
let mut config = self.config.lock().unwrap();
match r.req.req_type {
VIRTIO_MEM_REQ_PLUG => {
let size: u64 = r.req.nb_blocks as u64 * config.block_size as u64;
let resp_type = MemEpollHandler::virtio_mem_state_change_request(
StateChangeRequest {
config: *config,
addr: r.req.addr,
size,
nb_blocks: r.req.nb_blocks,
mem_state: &mut self.mem_state,
host_addr: self.host_addr,
host_fd: self.host_fd,
plug: true,
},
);
if resp_type == VIRTIO_MEM_RESP_ACK {
config.plugged_size += size;
}
MemEpollHandler::virtio_mem_send_response(
&mem,
resp_type,
0u16,
r.status_addr,
)
}
VIRTIO_MEM_REQ_UNPLUG => {
let size: u64 = r.req.nb_blocks as u64 * config.block_size as u64;
let resp_type = MemEpollHandler::virtio_mem_state_change_request(
StateChangeRequest {
config: *config,
addr: r.req.addr,
size,
nb_blocks: r.req.nb_blocks,
mem_state: &mut self.mem_state,
host_addr: self.host_addr,
host_fd: self.host_fd,
plug: false,
},
);
if resp_type == VIRTIO_MEM_RESP_ACK {
config.plugged_size -= size;
}
MemEpollHandler::virtio_mem_send_response(
&mem,
resp_type,
0u16,
r.status_addr,
)
}
VIRTIO_MEM_REQ_UNPLUG_ALL => {
let resp_type = MemEpollHandler::virtio_mem_unplug_all(
*config,
&mut self.mem_state,
self.host_addr,
self.host_fd,
);
if resp_type == VIRTIO_MEM_RESP_ACK {
config.plugged_size = 0;
config.usable_region_size = cmp::min(
config.region_size,
config.requested_size + VIRTIO_MEM_USABLE_EXTENT,
);
}
MemEpollHandler::virtio_mem_send_response(
&mem,
resp_type,
0u16,
r.status_addr,
)
}
VIRTIO_MEM_REQ_STATE => {
let (resp_type, resp_state) = MemEpollHandler::virtio_mem_state_request(
*config,
r.req.addr,
r.req.nb_blocks,
&mut self.mem_state,
);
MemEpollHandler::virtio_mem_send_response(
&mem,
resp_type,
resp_state,
r.status_addr,
)
}
_ => {
error!("VirtioMemReq unknown request type {:?}", r.req.req_type);
0
}
}
}
};
used_desc_heads[used_count] = (avail_desc.index, len);
used_count += 1;
}
for &(desc_index, len) in &used_desc_heads[..used_count] {
self.queue.add_used(&mem, desc_index, len);
}
used_count > 0
}
fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
// Create the epoll file descriptor
let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
// Use 'File' to enforce closing on 'epoll_fd'
let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
// Add events
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.resize.evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RESIZE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.queue_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(QUEUE_AVAIL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.kill_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.pause_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
const EPOLL_EVENTS_LEN: usize = 100;
let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
// Before jumping into the epoll loop, check if the device is expected
// to be in a paused state. This is helpful for the restore code path
// as the device thread should not start processing anything before the
// device has been resumed.
while paused.load(Ordering::SeqCst) {
thread::park();
}
'epoll: loop {
let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
Ok(res) => res,
Err(e) => {
if e.kind() == io::ErrorKind::Interrupted {
// It's well defined from the epoll_wait() syscall
// documentation that the epoll loop can be interrupted
// before any of the requested events occurred or the
// timeout expired. In both those cases, epoll_wait()
// returns an error of type EINTR, but this should not
// be considered as a regular error. Instead it is more
// appropriate to retry, by calling into epoll_wait().
continue;
}
return Err(DeviceError::EpollWait(e));
}
};
for event in events.iter().take(num_events) {
let ev_type = event.data as u16;
match ev_type {
RESIZE_EVENT => {
if let Err(e) = self.resize.evt.read() {
return Err(DeviceError::EpollHander(format!(
"Failed to get resize event: {:?}",
e
)));
} else {
let size = self.resize.get_size();
let mut config = self.config.lock().unwrap();
let mut signal_error = false;
let r = if config.requested_size == size {
Err(Error::ResizeInval(format!("Virtio-mem resize {} is same with current config.requested_size", size)))
} else if size > config.region_size {
let region_size = config.region_size;
Err(Error::ResizeInval(format!(
"Virtio-mem resize {} is bigger than config.region_size {}",
size, region_size
)))
} else if size % (config.block_size as u64) != 0 {
let block_size = config.block_size;
Err(Error::ResizeInval(format!(
"Virtio-mem resize {} is not aligned with config.block_size {}",
size, block_size
)))
} else {
config.requested_size = size;
let tmp_size = cmp::min(
config.region_size,
config.requested_size + VIRTIO_MEM_USABLE_EXTENT,
);
config.usable_region_size =
cmp::max(config.usable_region_size, tmp_size);
if let Err(e) = self.signal(&VirtioInterruptType::Config) {
signal_error = true;
Err(Error::ResizeTriggerFail(e))
} else {
Ok(())
}
};
if let Err(e) = &r {
// This error will send back to resize caller.
error!("Handle resize event get error: {:?}", e);
}
if let Err(e) = self.resize.send(r) {
return Err(DeviceError::EpollHander(format!(
"Sending \"resize\" generated error: {:?}",
e
)));
}
if signal_error {
return Err(DeviceError::EpollHander(String::from(
"Signal get error",
)));
}
}
}
QUEUE_AVAIL_EVENT => {
if let Err(e) = self.queue_evt.read() {
return Err(DeviceError::EpollHander(format!(
"Failed to get queue event: {:?}",
e
)));
} else if self.process_queue() {
if let Err(e) = self.signal(&VirtioInterruptType::Queue) {
return Err(DeviceError::EpollHander(format!(
"Failed to signal used queue: {:?}",
e
)));
}
}
}
KILL_EVENT => {
debug!("kill_evt received, stopping epoll loop");
break 'epoll;
}
PAUSE_EVENT => {
debug!("PAUSE_EVENT received, pausing virtio-pmem epoll loop");
// We loop here to handle spurious park() returns.
// Until we have not resumed, the paused boolean will
// be true.
while paused.load(Ordering::SeqCst) {
thread::park();
}
// Drain pause event after the device has been resumed.
// This ensures the pause event has been seen by each
// and every thread related to this virtio device.
let _ = self.pause_evt.read();
}
_ => {
return Err(DeviceError::EpollHander(String::from(
"Unknown event for virtio-mem",
)));
}
}
}
}
Ok(())
}
}
// Virtio device for exposing entropy to the guest OS through virtio.
pub struct Mem {
id: String,
resize: Resize,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
avail_features: u64,
pub acked_features: u64,
host_addr: u64,
host_fd: Option<RawFd>,
config: Arc<Mutex<VirtioMemConfig>>,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
paused: Arc<AtomicBool>,
}
impl Mem {
// Create a new virtio-mem device.
pub fn new(id: String, region: &Arc<GuestRegionMmap>, resize: Resize) -> io::Result<Mem> {
let region_len = region.len();
if region_len != region_len / VIRTIO_MEM_DEFAULT_BLOCK_SIZE * VIRTIO_MEM_DEFAULT_BLOCK_SIZE
{
return Err(io::Error::new(
io::ErrorKind::Other,
format!(
"Virtio-mem size is not aligned with {}",
VIRTIO_MEM_DEFAULT_BLOCK_SIZE
),
));
}
// Fixme: Not support VIRTIO_MEM_F_ACPI_PXM
let avail_features = 1u64 << VIRTIO_F_VERSION_1;
let mut config = VirtioMemConfig::default();
config.block_size = VIRTIO_MEM_DEFAULT_BLOCK_SIZE as u32;
config.addr = region.start_addr().raw_value();
config.region_size = region.len();
config.usable_region_size = cmp::min(
config.region_size,
config.requested_size + VIRTIO_MEM_USABLE_EXTENT,
);
let host_fd = if let Some(f_offset) = region.file_offset() {
Some(f_offset.file().as_raw_fd())
} else {
None
};
Ok(Mem {
id,
resize,
kill_evt: None,
pause_evt: None,
avail_features,
acked_features: 0u64,
host_addr: region.as_ptr() as u64,
host_fd,
config: Arc::new(Mutex::new(config)),
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
paused: Arc::new(AtomicBool::new(false)),
})
}
}
impl Drop for Mem {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
}
}
impl VirtioDevice for Mem {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_MEM as u32
}
fn queue_max_sizes(&self) -> &[u16] {
QUEUE_SIZES
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("Received acknowledge request for unknown feature.");
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, offset: u64, mut data: &mut [u8]) {
let config = self.config.lock().unwrap();
let config_slice = config.as_slice();
let config_len = config_slice.len() as u64;
if offset >= config_len {
error!("Failed to read config space");
return;
}
if let Some(end) = offset.checked_add(data.len() as u64) {
// This write can't fail, offset and end are checked against config_len.
data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
.unwrap();
}
}
fn write_config(&mut self, _offset: u64, _data: &[u8]) {
warn!("virtio-mem device configuration is read-only");
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
mut queues: Vec<Queue>,
mut queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
NUM_QUEUES,
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
let config = self.config.lock().unwrap();
let mut handler = MemEpollHandler {
host_addr: self.host_addr,
host_fd: self.host_fd,
mem_state: vec![false; config.region_size as usize / config.block_size as usize],
config: self.config.clone(),
resize: self.resize.try_clone().map_err(|e| {
error!("failed to clone resize EventFd: {:?}", e);
ActivateError::BadActivate
})?,
queue: queues.remove(0),
mem,
interrupt_cb,
queue_evt: queue_evts.remove(0),
kill_evt,
pause_evt,
};
let paused = self.paused.clone();
let mut epoll_threads = Vec::new();
thread::Builder::new()
.name("virtio_mem".to_string())
.spawn(move || handler.run(paused))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone virtio-mem epoll thread: {}", e);
ActivateError::BadActivate
})?;
self.epoll_threads = Some(epoll_threads);
Ok(())
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
}
virtio_pausable!(Mem);
impl Snapshottable for Mem {
fn id(&self) -> String {
self.id.clone()
}
}
impl Transportable for Mem {}
impl Migratable for Mem {}

813
vm-virtio/src/net.rs
View File

@@ -1,813 +0,0 @@
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the THIRD-PARTY file.
use super::net_util::{
build_net_config_space, build_net_config_space_with_mq, open_tap, register_listener,
unregister_listener, CtrlVirtio, NetCtrlEpollHandler, RxVirtio, TxVirtio, VirtioNetConfig,
KILL_EVENT, NET_EVENTS_COUNT, PAUSE_EVENT, RX_QUEUE_EVENT, RX_TAP_EVENT, TX_QUEUE_EVENT,
};
use super::Error as DeviceError;
use super::{
ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
};
use crate::VirtioInterrupt;
use anyhow::anyhow;
use libc::EAGAIN;
use libc::EFD_NONBLOCK;
use net_util::{MacAddr, Tap};
use std::cmp;
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::io::{self, Write};
use std::net::Ipv4Addr;
use std::num::Wrapping;
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::result;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::thread;
use std::vec::Vec;
use virtio_bindings::bindings::virtio_net::*;
use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
use vm_migration::{
Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
Transportable,
};
use vmm_sys_util::eventfd::EventFd;
#[derive(Debug)]
pub enum Error {
/// Failed to open taps.
OpenTap(super::net_util::Error),
}
pub type Result<T> = result::Result<T, Error>;
pub struct NetQueuePair {
pub mem: Option<GuestMemoryAtomic<GuestMemoryMmap>>,
pub tap: Tap,
pub rx: RxVirtio,
pub tx: TxVirtio,
pub epoll_fd: Option<RawFd>,
pub rx_tap_listening: bool,
pub counters: NetCounters,
}
impl NetQueuePair {
// Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
// if a buffer was used, and false if the frame must be deferred until a buffer
// is made available by the driver.
fn rx_single_frame(&mut self, mut queue: &mut Queue) -> result::Result<bool, DeviceError> {
let mem = self
.mem
.as_ref()
.ok_or(DeviceError::NoMemoryConfigured)
.map(|m| m.memory())?;
let next_desc = queue.iter(&mem).next();
if next_desc.is_none() {
// Queue has no available descriptors
if self.rx_tap_listening {
unregister_listener(
self.epoll_fd.unwrap(),
self.tap.as_raw_fd(),
epoll::Events::EPOLLIN,
u64::from(RX_TAP_EVENT),
)
.map_err(DeviceError::UnregisterListener)?;
self.rx_tap_listening = false;
info!("Listener unregistered");
}
return Ok(false);
}
Ok(self.rx.process_desc_chain(&mem, next_desc, &mut queue))
}
fn process_rx(&mut self, queue: &mut Queue) -> result::Result<bool, DeviceError> {
// Read as many frames as possible.
loop {
match self.read_tap() {
Ok(count) => {
self.rx.bytes_read = count;
if !self.rx_single_frame(queue)? {
self.rx.deferred_frame = true;
break;
}
}
Err(e) => {
// The tap device is non-blocking, so any error aside from EAGAIN is
// unexpected.
match e.raw_os_error() {
Some(err) if err == EAGAIN => (),
_ => {
error!("Failed to read tap: {:?}", e);
return Err(DeviceError::FailedReadTap);
}
};
break;
}
}
}
// Consume the counters from the Rx/Tx queues and accumulate into
// the counters for the device as whole. This consumption is needed
// to handle MQ.
self.counters
.rx_bytes
.fetch_add(self.rx.counter_bytes.0, Ordering::AcqRel);
self.counters
.rx_frames
.fetch_add(self.rx.counter_frames.0, Ordering::AcqRel);
self.rx.counter_bytes = Wrapping(0);
self.rx.counter_frames = Wrapping(0);
if self.rx.deferred_irqs {
self.rx.deferred_irqs = false;
let mem = self
.mem
.as_ref()
.ok_or(DeviceError::NoMemoryConfigured)
.map(|m| m.memory())?;
Ok(queue.needs_notification(&mem, queue.next_used))
} else {
Ok(false)
}
}
pub fn resume_rx(&mut self, queue: &mut Queue) -> result::Result<bool, DeviceError> {
if !self.rx_tap_listening {
register_listener(
self.epoll_fd.unwrap(),
self.tap.as_raw_fd(),
epoll::Events::EPOLLIN,
u64::from(RX_TAP_EVENT),
)
.map_err(DeviceError::RegisterListener)?;
self.rx_tap_listening = true;
info!("Listener registered");
}
if self.rx.deferred_frame {
if self.rx_single_frame(queue)? {
self.rx.deferred_frame = false;
// process_rx() was interrupted possibly before consuming all
// packets in the tap; try continuing now.
self.process_rx(queue)
} else if self.rx.deferred_irqs {
self.rx.deferred_irqs = false;
let mem = self
.mem
.as_ref()
.ok_or(DeviceError::NoMemoryConfigured)
.map(|m| m.memory())?;
Ok(queue.needs_notification(&mem, queue.next_used))
} else {
Ok(false)
}
} else {
Ok(false)
}
}
pub fn process_tx(&mut self, mut queue: &mut Queue) -> result::Result<bool, DeviceError> {
let mem = self
.mem
.as_ref()
.ok_or(DeviceError::NoMemoryConfigured)
.map(|m| m.memory())?;
self.tx.process_desc_chain(&mem, &mut self.tap, &mut queue);
self.counters
.tx_bytes
.fetch_add(self.tx.counter_bytes.0, Ordering::AcqRel);
self.counters
.tx_frames
.fetch_add(self.tx.counter_frames.0, Ordering::AcqRel);
self.tx.counter_bytes = Wrapping(0);
self.tx.counter_frames = Wrapping(0);
Ok(queue.needs_notification(&mem, queue.next_used))
}
pub fn process_rx_tap(&mut self, mut queue: &mut Queue) -> result::Result<bool, DeviceError> {
if self.rx.deferred_frame
// Process a deferred frame first if available. Don't read from tap again
// until we manage to receive this deferred frame.
{
if self.rx_single_frame(&mut queue)? {
self.rx.deferred_frame = false;
self.process_rx(&mut queue)
} else if self.rx.deferred_irqs {
self.rx.deferred_irqs = false;
Ok(true)
} else {
Ok(false)
}
} else {
self.process_rx(&mut queue)
}
}
fn read_tap(&mut self) -> io::Result<usize> {
self.tap.read(&mut self.rx.frame_buf)
}
}
struct NetEpollHandler {
net: NetQueuePair,
interrupt_cb: Arc<dyn VirtioInterrupt>,
kill_evt: EventFd,
pause_evt: EventFd,
// Always generate interrupts until the driver has signalled to the device.
// This mitigates a problem with interrupts from tap events being "lost" upon
// a restore as the vCPU thread isn't ready to handle the interrupt. This causes
// issues when combined with VIRTIO_RING_F_EVENT_IDX interrupt suppression.
driver_awake: bool,
}
impl NetEpollHandler {
fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
self.interrupt_cb
.trigger(&VirtioInterruptType::Queue, Some(queue))
.map_err(|e| {
error!("Failed to signal used queue: {:?}", e);
DeviceError::FailedSignalingUsedQueue(e)
})
}
fn handle_rx_event(
&mut self,
mut queue: &mut Queue,
queue_evt: &EventFd,
) -> result::Result<(), DeviceError> {
if let Err(e) = queue_evt.read() {
error!("Failed to get rx queue event: {:?}", e);
}
if self.net.resume_rx(&mut queue)? || !self.driver_awake {
self.signal_used_queue(queue)?;
info!("Signalling RX queue");
} else {
info!("Not signalling RX queue");
}
Ok(())
}
fn handle_tx_event(
&mut self,
mut queue: &mut Queue,
queue_evt: &EventFd,
) -> result::Result<(), DeviceError> {
if let Err(e) = queue_evt.read() {
error!("Failed to get tx queue event: {:?}", e);
}
if self.net.process_tx(&mut queue)? || !self.driver_awake {
self.signal_used_queue(queue)?;
info!("Signalling TX queue");
} else {
info!("Not signalling TX queue");
}
Ok(())
}
fn handle_rx_tap_event(&mut self, queue: &mut Queue) -> result::Result<(), DeviceError> {
if self.net.process_rx_tap(queue)? || !self.driver_awake {
self.signal_used_queue(queue)?;
info!("Signalling RX queue");
} else {
info!("Not signalling RX queue");
}
Ok(())
}
fn run(
&mut self,
paused: Arc<AtomicBool>,
mut queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
) -> result::Result<(), DeviceError> {
// Create the epoll file descriptor
let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
// Use 'File' to enforce closing on 'epoll_fd'
let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
self.net.epoll_fd = Some(epoll_fd);
// Add events
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
queue_evts[0].as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RX_QUEUE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
queue_evts[1].as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(TX_QUEUE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.kill_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.pause_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
// If there are some already available descriptors on the RX queue,
// then we can start the thread while listening onto the TAP.
if queues[0]
.available_descriptors(&self.net.mem.as_ref().unwrap().memory())
.unwrap()
{
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.net.tap.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RX_TAP_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
self.net.rx_tap_listening = true;
error!("Listener registered at start");
}
let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); NET_EVENTS_COUNT];
// Before jumping into the epoll loop, check if the device is expected
// to be in a paused state. This is helpful for the restore code path
// as the device thread should not start processing anything before the
// device has been resumed.
while paused.load(Ordering::SeqCst) {
thread::park();
}
'epoll: loop {
let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
Ok(res) => res,
Err(e) => {
if e.kind() == io::ErrorKind::Interrupted {
// It's well defined from the epoll_wait() syscall
// documentation that the epoll loop can be interrupted
// before any of the requested events occurred or the
// timeout expired. In both those cases, epoll_wait()
// returns an error of type EINTR, but this should not
// be considered as a regular error. Instead it is more
// appropriate to retry, by calling into epoll_wait().
continue;
}
return Err(DeviceError::EpollWait(e));
}
};
for event in events.iter().take(num_events) {
let ev_type = event.data as u16;
match ev_type {
RX_QUEUE_EVENT => {
self.driver_awake = true;
self.handle_rx_event(&mut queues[0], &queue_evts[0])?;
}
TX_QUEUE_EVENT => {
self.driver_awake = true;
self.handle_tx_event(&mut queues[1], &queue_evts[1])?;
}
RX_TAP_EVENT => {
self.handle_rx_tap_event(&mut queues[0])?;
}
KILL_EVENT => {
debug!("KILL_EVENT received, stopping epoll loop");
break 'epoll;
}
PAUSE_EVENT => {
debug!("PAUSE_EVENT received, pausing virtio-net epoll loop");
// We loop here to handle spurious park() returns.
// Until we have not resumed, the paused boolean will
// be true.
while paused.load(Ordering::SeqCst) {
thread::park();
}
// Drain pause event after the device has been resumed.
// This ensures the pause event has been seen by each
// and every thread related to this virtio device.
let _ = self.pause_evt.read();
}
_ => {
error!("Unknown event for virtio-net");
}
}
}
}
Ok(())
}
}
#[derive(Default, Clone)]
pub struct NetCounters {
tx_bytes: Arc<AtomicU64>,
tx_frames: Arc<AtomicU64>,
rx_bytes: Arc<AtomicU64>,
rx_frames: Arc<AtomicU64>,
}
pub struct Net {
id: String,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
taps: Option<Vec<Tap>>,
avail_features: u64,
acked_features: u64,
config: VirtioNetConfig,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
ctrl_queue_epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
paused: Arc<AtomicBool>,
queue_size: Vec<u16>,
counters: NetCounters,
}
#[derive(Serialize, Deserialize)]
pub struct NetState {
pub avail_features: u64,
pub acked_features: u64,
pub config: VirtioNetConfig,
pub queue_size: Vec<u16>,
}
impl Net {
/// Create a new virtio network device with the given TAP interface.
pub fn new_with_tap(
id: String,
taps: Vec<Tap>,
guest_mac: Option<MacAddr>,
iommu: bool,
num_queues: usize,
queue_size: u16,
) -> Result<Self> {
let mut avail_features = 1 << VIRTIO_NET_F_GUEST_CSUM
| 1 << VIRTIO_NET_F_CSUM
| 1 << VIRTIO_NET_F_GUEST_TSO4
| 1 << VIRTIO_NET_F_GUEST_UFO
| 1 << VIRTIO_NET_F_HOST_TSO4
| 1 << VIRTIO_NET_F_HOST_UFO
| 1 << VIRTIO_RING_F_EVENT_IDX
| 1 << VIRTIO_F_VERSION_1;
if iommu {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ;
let queue_num = num_queues + 1;
let mut config = VirtioNetConfig::default();
if let Some(mac) = guest_mac {
build_net_config_space(&mut config, mac, num_queues, &mut avail_features);
} else {
build_net_config_space_with_mq(&mut config, num_queues, &mut avail_features);
}
Ok(Net {
id,
kill_evt: None,
pause_evt: None,
taps: Some(taps),
avail_features,
acked_features: 0u64,
config,
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
ctrl_queue_epoll_thread: None,
paused: Arc::new(AtomicBool::new(false)),
queue_size: vec![queue_size; queue_num],
counters: NetCounters::default(),
})
}
/// Create a new virtio network device with the given IP address and
/// netmask.
#[allow(clippy::too_many_arguments)]
pub fn new(
id: String,
if_name: Option<&str>,
ip_addr: Option<Ipv4Addr>,
netmask: Option<Ipv4Addr>,
guest_mac: Option<MacAddr>,
host_mac: &mut Option<MacAddr>,
iommu: bool,
num_queues: usize,
queue_size: u16,
) -> Result<Self> {
let taps = open_tap(if_name, ip_addr, netmask, host_mac, num_queues / 2)
.map_err(Error::OpenTap)?;
Self::new_with_tap(id, taps, guest_mac, iommu, num_queues, queue_size)
}
fn state(&self) -> NetState {
NetState {
avail_features: self.avail_features,
acked_features: self.acked_features,
config: self.config,
queue_size: self.queue_size.clone(),
}
}
fn set_state(&mut self, state: &NetState) -> Result<()> {
self.avail_features = state.avail_features;
self.acked_features = state.acked_features;
self.config = state.config;
self.queue_size = state.queue_size.clone();
Ok(())
}
}
impl Drop for Net {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
}
}
impl VirtioDevice for Net {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_NET as u32
}
fn queue_max_sizes(&self) -> &[u16] {
&self.queue_size.as_slice()
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("Received acknowledge request for unknown feature: {:x}", v);
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, offset: u64, mut data: &mut [u8]) {
let config_slice = self.config.as_slice();
let config_len = config_slice.len() as u64;
if offset >= config_len {
error!("Failed to read config space");
return;
}
if let Some(end) = offset.checked_add(data.len() as u64) {
// This write can't fail, offset and end are checked against config_len.
data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
.unwrap();
}
}
fn write_config(&mut self, offset: u64, data: &[u8]) {
let config_slice = self.config.as_mut_slice();
let data_len = data.len() as u64;
let config_len = config_slice.len() as u64;
if offset + data_len > config_len {
error!("Failed to write config space");
return;
}
let (_, right) = config_slice.split_at_mut(offset as usize);
right.copy_from_slice(&data[..]);
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
mut queues: Vec<Queue>,
mut queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != self.queue_size.len() || queue_evts.len() != self.queue_size.len() {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
self.queue_size.len(),
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
if let Some(mut taps) = self.taps.clone() {
// Save the interrupt EventFD as we need to return it on reset
// but clone it to pass into the thread.
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
let queue_num = queues.len();
if (self.acked_features & 1 << VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0 {
let cvq_queue = queues.remove(queue_num - 1);
let cvq_queue_evt = queue_evts.remove(queue_num - 1);
let mut ctrl_handler = NetCtrlEpollHandler {
mem: mem.clone(),
kill_evt: kill_evt.try_clone().unwrap(),
pause_evt: pause_evt.try_clone().unwrap(),
ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
epoll_fd: 0,
};
let paused = self.paused.clone();
thread::Builder::new()
.name("virtio_net".to_string())
.spawn(move || ctrl_handler.run_ctrl(paused))
.map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
.map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?;
}
let event_idx = self.acked_features & 1 << VIRTIO_RING_F_EVENT_IDX != 0;
let mut epoll_threads = Vec::new();
for _ in 0..taps.len() {
let rx = RxVirtio::new();
let tx = TxVirtio::new();
let rx_tap_listening = false;
let mut queue_pair = Vec::new();
queue_pair.push(queues.remove(0));
queue_pair.push(queues.remove(0));
queue_pair[0].set_event_idx(event_idx);
queue_pair[1].set_event_idx(event_idx);
let mut queue_evt_pair = Vec::new();
queue_evt_pair.push(queue_evts.remove(0));
queue_evt_pair.push(queue_evts.remove(0));
let mut handler = NetEpollHandler {
net: NetQueuePair {
mem: Some(mem.clone()),
tap: taps.remove(0),
rx,
tx,
epoll_fd: None,
rx_tap_listening,
counters: self.counters.clone(),
},
interrupt_cb: interrupt_cb.clone(),
kill_evt: kill_evt.try_clone().unwrap(),
pause_evt: pause_evt.try_clone().unwrap(),
driver_awake: false,
};
let paused = self.paused.clone();
thread::Builder::new()
.name("virtio_net".to_string())
.spawn(move || handler.run(paused, queue_pair, queue_evt_pair))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?;
}
self.epoll_threads = Some(epoll_threads);
return Ok(());
}
Err(ActivateError::BadActivate)
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
fn counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>> {
let mut counters = HashMap::new();
counters.insert(
"rx_bytes",
Wrapping(self.counters.rx_bytes.load(Ordering::Acquire)),
);
counters.insert(
"rx_frames",
Wrapping(self.counters.rx_frames.load(Ordering::Acquire)),
);
counters.insert(
"tx_bytes",
Wrapping(self.counters.tx_bytes.load(Ordering::Acquire)),
);
counters.insert(
"tx_frames",
Wrapping(self.counters.tx_frames.load(Ordering::Acquire)),
);
Some(counters)
}
}
virtio_ctrl_q_pausable!(Net);
impl Snapshottable for Net {
fn id(&self) -> String {
self.id.clone()
}
fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
let snapshot =
serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
let mut net_snapshot = Snapshot::new(self.id.as_str());
net_snapshot.add_data_section(SnapshotDataSection {
id: format!("{}-section", self.id),
snapshot,
});
Ok(net_snapshot)
}
fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
if let Some(net_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
let net_state = match serde_json::from_slice(&net_section.snapshot) {
Ok(state) => state,
Err(error) => {
return Err(MigratableError::Restore(anyhow!(
"Could not deserialize NET {}",
error
)))
}
};
return self.set_state(&net_state).map_err(|e| {
MigratableError::Restore(anyhow!("Could not restore NET state {:?}", e))
});
}
Err(MigratableError::Restore(anyhow!(
"Could not find NET snapshot section"
)))
}
}
impl Transportable for Net {}
impl Migratable for Net {}

622
vm-virtio/src/net_util.rs
View File

@@ -1,622 +0,0 @@
// Copyright (c) 2019 Intel Corporation. All rights reserved.
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
use super::Error as DeviceError;
use super::{DescriptorChain, DeviceEventT, Queue};
use net_util::{MacAddr, Tap, TapError};
use serde::ser::{Serialize, SerializeStruct, Serializer};
use std::cmp;
use std::fs;
use std::fs::File;
use std::io::{self, Write};
use std::mem;
use std::net::Ipv4Addr;
use std::num::Wrapping;
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use virtio_bindings::bindings::virtio_net::*;
use vm_memory::{
ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryError,
GuestMemoryMmap,
};
use vmm_sys_util::eventfd::EventFd;
type Result<T> = std::result::Result<T, Error>;
/// The maximum buffer size when segmentation offload is enabled. This
/// includes the 12-byte virtio net header.
/// http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html#x1-1740003
const MAX_BUFFER_SIZE: usize = 65562;
const QUEUE_SIZE: usize = 256;
// The guest has made a buffer available to receive a frame into.
pub const RX_QUEUE_EVENT: DeviceEventT = 0;
// The transmit queue has a frame that is ready to send from the guest.
pub const TX_QUEUE_EVENT: DeviceEventT = 1;
// A frame is available for reading from the tap device to receive in the guest.
pub const RX_TAP_EVENT: DeviceEventT = 2;
// The device has been dropped.
pub const KILL_EVENT: DeviceEventT = 3;
// The device should be paused.
pub const PAUSE_EVENT: DeviceEventT = 4;
// Number of DeviceEventT events supported by this implementation.
pub const NET_EVENTS_COUNT: usize = 5;
// The device has been dropped.
const CTRL_QUEUE_EVENT: DeviceEventT = 0;
// Number of DeviceEventT events supported by this implementation.
const CTRL_EVENT_COUNT: usize = 3;
#[repr(C, packed)]
#[derive(Copy, Clone, Debug, Default, Deserialize)]
pub struct VirtioNetConfig {
pub mac: [u8; 6],
pub status: u16,
pub max_virtqueue_pairs: u16,
pub mtu: u16,
pub speed: u32,
pub duplex: u8,
}
// We must explicitly implement Serialize since the structure is packed and
// it's unsafe to borrow from a packed structure. And by default, if we derive
// Serialize from serde, it will borrow the values from the structure.
// That's why this implementation copies each field separately before it
// serializes the entire structure field by field.
impl Serialize for VirtioNetConfig {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
{
let mac = self.mac;
let status = self.status;
let max_virtqueue_pairs = self.max_virtqueue_pairs;
let mtu = self.mtu;
let speed = self.speed;
let duplex = self.duplex;
let mut virtio_net_config = serializer.serialize_struct("VirtioNetConfig", 17)?;
virtio_net_config.serialize_field("mac", &mac)?;
virtio_net_config.serialize_field("status", &status)?;
virtio_net_config.serialize_field("max_virtqueue_pairs", &max_virtqueue_pairs)?;
virtio_net_config.serialize_field("mtu", &mtu)?;
virtio_net_config.serialize_field("speed", &speed)?;
virtio_net_config.serialize_field("duplex", &duplex)?;
virtio_net_config.end()
}
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioNetConfig {}
#[derive(Debug)]
pub enum Error {
/// Failed to convert an hexadecimal string into an integer.
ConvertHexStringToInt(std::num::ParseIntError),
/// Read process MQ.
FailedProcessMQ,
/// Read queue failed.
GuestMemory(GuestMemoryError),
/// Invalid ctrl class
InvalidCtlClass,
/// Invalid ctrl command
InvalidCtlCmd,
/// Invalid descriptor
InvalidDesc,
/// Invalid queue pairs number
InvalidQueuePairsNum,
/// Error related to the multiqueue support.
MultiQueueSupport(String),
/// No memory passed in.
NoMemory,
/// No ueue pairs nummber.
NoQueuePairsNum,
/// Failed to read the TAP flags from sysfs.
ReadSysfsTunFlags(io::Error),
/// Open tap device failed.
TapOpen(TapError),
/// Setting tap IP failed.
TapSetIp(TapError),
/// Setting tap netmask failed.
TapSetNetmask(TapError),
/// Setting MAC address failed
TapSetMac(TapError),
/// Getting MAC address failed
TapGetMac(TapError),
/// Setting tap interface offload flags failed.
TapSetOffload(TapError),
/// Setting vnet header size failed.
TapSetVnetHdrSize(TapError),
/// Enabling tap interface failed.
TapEnable(TapError),
}
pub struct CtrlVirtio {
pub queue_evt: EventFd,
pub queue: Queue,
}
impl std::clone::Clone for CtrlVirtio {
fn clone(&self) -> Self {
CtrlVirtio {
queue_evt: self.queue_evt.try_clone().unwrap(),
queue: self.queue.clone(),
}
}
}
impl CtrlVirtio {
pub fn new(queue: Queue, queue_evt: EventFd) -> Self {
CtrlVirtio { queue_evt, queue }
}
fn process_mq(&self, mem: &GuestMemoryMmap, avail_desc: DescriptorChain) -> Result<()> {
let mq_desc = if avail_desc.has_next() {
avail_desc.next_descriptor().unwrap()
} else {
return Err(Error::NoQueuePairsNum);
};
let queue_pairs = mem
.read_obj::<u16>(mq_desc.addr)
.map_err(Error::GuestMemory)?;
if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
|| (queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
{
return Err(Error::InvalidQueuePairsNum);
}
let status_desc = if mq_desc.has_next() {
mq_desc.next_descriptor().unwrap()
} else {
return Err(Error::NoQueuePairsNum);
};
mem.write_obj::<u8>(0, status_desc.addr)
.map_err(Error::GuestMemory)?;
Ok(())
}
pub fn process_cvq(&mut self, mem: &GuestMemoryMmap) -> Result<()> {
let mut used_desc_heads = [(0, 0); QUEUE_SIZE];
let mut used_count = 0;
if let Some(avail_desc) = self.queue.iter(&mem).next() {
used_desc_heads[used_count] = (avail_desc.index, avail_desc.len);
used_count += 1;
let ctrl_hdr = mem
.read_obj::<u16>(avail_desc.addr)
.map_err(Error::GuestMemory)?;
let ctrl_hdr_v = ctrl_hdr.as_slice();
let class = ctrl_hdr_v[0];
let cmd = ctrl_hdr_v[1];
match u32::from(class) {
VIRTIO_NET_CTRL_MQ => {
if u32::from(cmd) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET {
return Err(Error::InvalidCtlCmd);
}
if let Err(_e) = self.process_mq(&mem, avail_desc) {
return Err(Error::FailedProcessMQ);
}
}
_ => return Err(Error::InvalidCtlClass),
}
} else {
return Err(Error::InvalidDesc);
}
for &(desc_index, len) in &used_desc_heads[..used_count] {
self.queue.add_used(&mem, desc_index, len);
self.queue.update_avail_event(&mem);
}
Ok(())
}
}
pub fn register_listener(
epoll_fd: RawFd,
fd: RawFd,
ev_type: epoll::Events,
data: u64,
) -> std::result::Result<(), io::Error> {
epoll::ctl(
epoll_fd,
epoll::ControlOptions::EPOLL_CTL_ADD,
fd,
epoll::Event::new(ev_type, data),
)
}
pub fn unregister_listener(
epoll_fd: RawFd,
fd: RawFd,
ev_type: epoll::Events,
data: u64,
) -> std::result::Result<(), io::Error> {
epoll::ctl(
epoll_fd,
epoll::ControlOptions::EPOLL_CTL_DEL,
fd,
epoll::Event::new(ev_type, data),
)
}
pub struct NetCtrlEpollHandler {
pub mem: GuestMemoryAtomic<GuestMemoryMmap>,
pub kill_evt: EventFd,
pub pause_evt: EventFd,
pub ctrl_q: CtrlVirtio,
pub epoll_fd: RawFd,
}
impl NetCtrlEpollHandler {
pub fn run_ctrl(&mut self, paused: Arc<AtomicBool>) -> std::result::Result<(), DeviceError> {
// Create the epoll file descriptor
self.epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
// Use 'File' to enforce closing on 'epoll_fd'
let epoll_file = unsafe { File::from_raw_fd(self.epoll_fd) };
register_listener(
epoll_file.as_raw_fd(),
self.ctrl_q.queue_evt.as_raw_fd(),
epoll::Events::EPOLLIN,
u64::from(CTRL_QUEUE_EVENT),
)
.unwrap();
register_listener(
epoll_file.as_raw_fd(),
self.kill_evt.as_raw_fd(),
epoll::Events::EPOLLIN,
u64::from(KILL_EVENT),
)
.unwrap();
register_listener(
epoll_file.as_raw_fd(),
self.pause_evt.as_raw_fd(),
epoll::Events::EPOLLIN,
u64::from(PAUSE_EVENT),
)
.unwrap();
let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); CTRL_EVENT_COUNT];
// Before jumping into the epoll loop, check if the device is expected
// to be in a paused state. This is helpful for the restore code path
// as the device thread should not start processing anything before the
// device has been resumed.
while paused.load(Ordering::SeqCst) {
thread::park();
}
'epoll: loop {
let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
Ok(res) => res,
Err(e) => {
if e.kind() == std::io::ErrorKind::Interrupted {
continue;
}
return Err(DeviceError::EpollWait(e));
}
};
for event in events.iter().take(num_events) {
let ev_type = event.data as u16;
match ev_type {
CTRL_QUEUE_EVENT => {
let mem = self.mem.memory();
if let Err(e) = self.ctrl_q.queue_evt.read() {
error!("failed to get ctl queue event: {:?}", e);
}
if let Err(e) = self.ctrl_q.process_cvq(&mem) {
error!("failed to process ctrl queue: {:?}", e);
}
}
KILL_EVENT => {
break 'epoll;
}
PAUSE_EVENT => {
debug!("PAUSE_EVENT received, pausing vhost-user epoll loop");
// We loop here to handle spurious park() returns.
// Until we have not resumed, the paused boolean will
// be true.
while paused.load(Ordering::SeqCst) {
std::thread::park();
}
// Drain pause event after the device has been resumed.
// This ensures the pause event has been seen by each
// and every thread related to this virtio device.
let _ = self.pause_evt.read();
}
_ => {
error!("Unknown event for virtio-net");
}
}
}
}
Ok(())
}
}
#[derive(Clone)]
pub struct TxVirtio {
pub iovec: Vec<(GuestAddress, usize)>,
pub frame_buf: [u8; MAX_BUFFER_SIZE],
pub counter_bytes: Wrapping<u64>,
pub counter_frames: Wrapping<u64>,
}
impl Default for TxVirtio {
fn default() -> Self {
Self::new()
}
}
impl TxVirtio {
pub fn new() -> Self {
TxVirtio {
iovec: Vec::new(),
frame_buf: [0u8; MAX_BUFFER_SIZE],
counter_bytes: Wrapping(0),
counter_frames: Wrapping(0),
}
}
pub fn process_desc_chain(&mut self, mem: &GuestMemoryMmap, tap: &mut Tap, queue: &mut Queue) {
while let Some(avail_desc) = queue.iter(&mem).next() {
let head_index = avail_desc.index;
let mut read_count = 0;
let mut next_desc = Some(avail_desc);
self.iovec.clear();
while let Some(desc) = next_desc {
if desc.is_write_only() {
break;
}
self.iovec.push((desc.addr, desc.len as usize));
read_count += desc.len as usize;
next_desc = desc.next_descriptor();
}
read_count = 0;
// Copy buffer from across multiple descriptors.
// TODO(performance - Issue #420): change this to use `writev()` instead of `write()`
// and get rid of the intermediate buffer.
for (desc_addr, desc_len) in self.iovec.drain(..) {
let limit = cmp::min((read_count + desc_len) as usize, self.frame_buf.len());
let read_result =
mem.read_slice(&mut self.frame_buf[read_count..limit as usize], desc_addr);
match read_result {
Ok(_) => {
// Increment by number of bytes actually read
read_count += limit - read_count;
}
Err(e) => {
println!("Failed to read slice: {:?}", e);
break;
}
}
}
let write_result = tap.write(&self.frame_buf[..read_count]);
match write_result {
Ok(_) => {}
Err(e) => {
println!("net: tx: error failed to write to tap: {}", e);
}
};
self.counter_bytes += Wrapping((read_count - vnet_hdr_len()) as u64);
self.counter_frames += Wrapping(1);
queue.add_used(&mem, head_index, 0);
queue.update_avail_event(&mem);
}
}
}
#[derive(Clone)]
pub struct RxVirtio {
pub deferred_frame: bool,
pub deferred_irqs: bool,
pub bytes_read: usize,
pub frame_buf: [u8; MAX_BUFFER_SIZE],
pub counter_bytes: Wrapping<u64>,
pub counter_frames: Wrapping<u64>,
}
impl Default for RxVirtio {
fn default() -> Self {
Self::new()
}
}
impl RxVirtio {
pub fn new() -> Self {
RxVirtio {
deferred_frame: false,
deferred_irqs: false,
bytes_read: 0,
frame_buf: [0u8; MAX_BUFFER_SIZE],
counter_bytes: Wrapping(0),
counter_frames: Wrapping(0),
}
}
pub fn process_desc_chain(
&mut self,
mem: &GuestMemoryMmap,
mut next_desc: Option<DescriptorChain>,
queue: &mut Queue,
) -> bool {
let head_index = next_desc.as_ref().unwrap().index;
let mut write_count = 0;
// Copy from frame into buffer, which may span multiple descriptors.
loop {
match next_desc {
Some(desc) => {
if !desc.is_write_only() {
break;
}
let limit = cmp::min(write_count + desc.len as usize, self.bytes_read);
let source_slice = &self.frame_buf[write_count..limit];
let write_result = mem.write_slice(source_slice, desc.addr);
match write_result {
Ok(_) => {
write_count = limit;
}
Err(e) => {
error!("Failed to write slice: {:?}", e);
break;
}
};
if write_count >= self.bytes_read {
break;
}
next_desc = desc.next_descriptor();
}
None => {
warn!("Receiving buffer is too small to hold frame of current size");
break;
}
}
}
self.counter_bytes += Wrapping((write_count - vnet_hdr_len()) as u64);
self.counter_frames += Wrapping(1);
queue.add_used(&mem, head_index, write_count as u32);
queue.update_avail_event(&mem);
// Mark that we have at least one pending packet and we need to interrupt the guest.
self.deferred_irqs = true;
// Update the frame_buf buffer.
if write_count < self.bytes_read {
self.frame_buf.copy_within(write_count..self.bytes_read, 0);
self.bytes_read -= write_count;
false
} else {
self.bytes_read = 0;
true
}
}
}
pub fn build_net_config_space(
mut config: &mut VirtioNetConfig,
mac: MacAddr,
num_queues: usize,
mut avail_features: &mut u64,
) {
config.mac.copy_from_slice(mac.get_bytes());
*avail_features |= 1 << VIRTIO_NET_F_MAC;
build_net_config_space_with_mq(&mut config, num_queues, &mut avail_features);
}
pub fn build_net_config_space_with_mq(
config: &mut VirtioNetConfig,
num_queues: usize,
avail_features: &mut u64,
) {
let num_queue_pairs = (num_queues / 2) as u16;
if (num_queue_pairs >= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
&& (num_queue_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
{
config.max_virtqueue_pairs = num_queue_pairs;
*avail_features |= 1 << VIRTIO_NET_F_MQ;
}
}
fn vnet_hdr_len() -> usize {
mem::size_of::<virtio_net_hdr_v1>()
}
fn check_mq_support(if_name: &Option<&str>, queue_pairs: usize) -> Result<()> {
if let Some(tap_name) = if_name {
let mq = queue_pairs > 1;
let path = format!("/sys/class/net/{}/tun_flags", tap_name);
// interface does not exist, check is not required
if !Path::new(&path).exists() {
return Ok(());
}
let tun_flags_str = fs::read_to_string(path).map_err(Error::ReadSysfsTunFlags)?;
let tun_flags = u32::from_str_radix(tun_flags_str.trim().trim_start_matches("0x"), 16)
.map_err(Error::ConvertHexStringToInt)?;
if (tun_flags & net_gen::IFF_MULTI_QUEUE != 0) && !mq {
return Err(Error::MultiQueueSupport(String::from(
"TAP interface supports MQ while device does not",
)));
} else if (tun_flags & net_gen::IFF_MULTI_QUEUE == 0) && mq {
return Err(Error::MultiQueueSupport(String::from(
"Device supports MQ while TAP interface does not",
)));
}
}
Ok(())
}
/// Create a new virtio network device with the given IP address and
/// netmask.
pub fn open_tap(
if_name: Option<&str>,
ip_addr: Option<Ipv4Addr>,
netmask: Option<Ipv4Addr>,
host_mac: &mut Option<MacAddr>,
num_rx_q: usize,
) -> Result<Vec<Tap>> {
let mut taps: Vec<Tap> = Vec::new();
let mut ifname: String = String::new();
let vnet_hdr_size = vnet_hdr_len() as i32;
let flag = net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6;
// In case the tap interface already exists, check if the number of
// queues is appropriate. The tap might not support multiqueue while
// the number of queues indicates the user expects multiple queues, or
// on the contrary, the tap might support multiqueue while the number
// of queues indicates the user doesn't expect multiple queues.
check_mq_support(&if_name, num_rx_q)?;
for i in 0..num_rx_q {
let tap: Tap;
if i == 0 {
tap = match if_name {
Some(name) => Tap::open_named(name, num_rx_q).map_err(Error::TapOpen)?,
None => Tap::new(num_rx_q).map_err(Error::TapOpen)?,
};
if let Some(ip) = ip_addr {
tap.set_ip_addr(ip).map_err(Error::TapSetIp)?;
}
if let Some(mask) = netmask {
tap.set_netmask(mask).map_err(Error::TapSetNetmask)?;
}
if let Some(mac) = host_mac {
tap.set_mac_addr(*mac).map_err(Error::TapSetMac)?
} else {
*host_mac = Some(tap.get_mac_addr().map_err(Error::TapGetMac)?)
}
tap.enable().map_err(Error::TapEnable)?;
tap.set_offload(flag).map_err(Error::TapSetOffload)?;
tap.set_vnet_hdr_size(vnet_hdr_size)
.map_err(Error::TapSetVnetHdrSize)?;
ifname = String::from_utf8(tap.get_if_name()).unwrap();
} else {
tap = Tap::open_named(ifname.as_str(), num_rx_q).map_err(Error::TapOpen)?;
tap.set_offload(flag).map_err(Error::TapSetOffload)?;
tap.set_vnet_hdr_size(vnet_hdr_size)
.map_err(Error::TapSetVnetHdrSize)?;
}
taps.push(tap);
}
Ok(taps)
}

631
vm-virtio/src/pmem.rs
View File

@@ -1,631 +0,0 @@
// Copyright 2019 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// Copyright © 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
use super::Error as DeviceError;
use super::{
ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, UserspaceMapping,
VirtioDevice, VirtioDeviceType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
};
use crate::{VirtioInterrupt, VirtioInterruptType};
use anyhow::anyhow;
use libc::EFD_NONBLOCK;
use serde::ser::{Serialize, SerializeStruct, Serializer};
use std::cmp;
use std::fmt::{self, Display};
use std::fs::File;
use std::io::{self, Write};
use std::mem::size_of;
use std::os::unix::io::{AsRawFd, FromRawFd};
use std::result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use vm_memory::{
Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
GuestMemoryError, GuestMemoryMmap, MmapRegion,
};
use vm_migration::{
Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
Transportable,
};
use vmm_sys_util::eventfd::EventFd;
const QUEUE_SIZE: u16 = 256;
const NUM_QUEUES: usize = 1;
const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE];
const VIRTIO_PMEM_REQ_TYPE_FLUSH: u32 = 0;
const VIRTIO_PMEM_RESP_TYPE_OK: u32 = 0;
const VIRTIO_PMEM_RESP_TYPE_EIO: u32 = 1;
// New descriptors are pending on the virtio queue.
const QUEUE_AVAIL_EVENT: DeviceEventT = 0;
// The device has been dropped.
const KILL_EVENT: DeviceEventT = 1;
// The device should be paused.
const PAUSE_EVENT: DeviceEventT = 2;
#[derive(Copy, Clone, Debug, Default, Deserialize)]
#[repr(C, packed)]
struct VirtioPmemConfig {
start: u64,
size: u64,
}
// We must explicitly implement Serialize since the structure is packed and
// it's unsafe to borrow from a packed structure. And by default, if we derive
// Serialize from serde, it will borrow the values from the structure.
// That's why this implementation copies each field separately before it
// serializes the entire structure field by field.
impl Serialize for VirtioPmemConfig {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let start = self.start;
let size = self.size;
let mut virtio_pmem_config = serializer.serialize_struct("VirtioPmemConfig", 16)?;
virtio_pmem_config.serialize_field("start", &start)?;
virtio_pmem_config.serialize_field("size", &size)?;
virtio_pmem_config.end()
}
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioPmemConfig {}
#[derive(Copy, Clone, Debug, Default)]
#[repr(C)]
struct VirtioPmemReq {
type_: u32,
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioPmemReq {}
#[derive(Copy, Clone, Debug, Default)]
#[repr(C)]
struct VirtioPmemResp {
ret: u32,
}
// Safe because it only has data and has no implicit padding.
unsafe impl ByteValued for VirtioPmemResp {}
#[derive(Debug)]
enum Error {
/// Guest gave us bad memory addresses.
GuestMemory(GuestMemoryError),
/// Guest gave us a write only descriptor that protocol says to read from.
UnexpectedWriteOnlyDescriptor,
/// Guest gave us a read only descriptor that protocol says to write to.
UnexpectedReadOnlyDescriptor,
/// Guest gave us too few descriptors in a descriptor chain.
DescriptorChainTooShort,
/// Guest gave us a buffer that was too short to use.
BufferLengthTooSmall,
/// Guest sent us invalid request.
InvalidRequest,
}
impl Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
use self::Error::*;
match self {
BufferLengthTooSmall => write!(f, "buffer length too small"),
DescriptorChainTooShort => write!(f, "descriptor chain too short"),
GuestMemory(e) => write!(f, "bad guest memory address: {}", e),
InvalidRequest => write!(f, "invalid request"),
UnexpectedReadOnlyDescriptor => write!(f, "unexpected read-only descriptor"),
UnexpectedWriteOnlyDescriptor => write!(f, "unexpected write-only descriptor"),
}
}
}
#[derive(Debug, PartialEq)]
enum RequestType {
Flush,
}
struct Request {
type_: RequestType,
status_addr: GuestAddress,
}
impl Request {
fn parse(
avail_desc: &DescriptorChain,
mem: &GuestMemoryMmap,
) -> result::Result<Request, Error> {
// The head contains the request type which MUST be readable.
if avail_desc.is_write_only() {
return Err(Error::UnexpectedWriteOnlyDescriptor);
}
if avail_desc.len as usize != size_of::<VirtioPmemReq>() {
return Err(Error::InvalidRequest);
}
let request: VirtioPmemReq = mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?;
let request_type = match request.type_ {
VIRTIO_PMEM_REQ_TYPE_FLUSH => RequestType::Flush,
_ => return Err(Error::InvalidRequest),
};
let status_desc = avail_desc
.next_descriptor()
.ok_or(Error::DescriptorChainTooShort)?;
// The status MUST always be writable
if !status_desc.is_write_only() {
return Err(Error::UnexpectedReadOnlyDescriptor);
}
if (status_desc.len as usize) < size_of::<VirtioPmemResp>() {
return Err(Error::BufferLengthTooSmall);
}
Ok(Request {
type_: request_type,
status_addr: status_desc.addr,
})
}
}
struct PmemEpollHandler {
queue: Queue,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
disk: File,
interrupt_cb: Arc<dyn VirtioInterrupt>,
queue_evt: EventFd,
kill_evt: EventFd,
pause_evt: EventFd,
}
impl PmemEpollHandler {
fn process_queue(&mut self) -> bool {
let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
let mut used_count = 0;
let mem = self.mem.memory();
for avail_desc in self.queue.iter(&mem) {
let len = match Request::parse(&avail_desc, &mem) {
Ok(ref req) if (req.type_ == RequestType::Flush) => {
let status_code = match self.disk.sync_all() {
Ok(()) => VIRTIO_PMEM_RESP_TYPE_OK,
Err(e) => {
error!("failed flushing disk image: {}", e);
VIRTIO_PMEM_RESP_TYPE_EIO
}
};
let resp = VirtioPmemResp { ret: status_code };
match mem.write_obj(resp, req.status_addr) {
Ok(_) => size_of::<VirtioPmemResp>() as u32,
Err(e) => {
error!("bad guest memory address: {}", e);
0
}
}
}
Ok(ref req) => {
// Currently, there is only one virtio-pmem request, FLUSH.
error!("Invalid virtio request type {:?}", req.type_);
0
}
Err(e) => {
error!("Failed to parse available descriptor chain: {:?}", e);
0
}
};
used_desc_heads[used_count] = (avail_desc.index, len);
used_count += 1;
}
for &(desc_index, len) in &used_desc_heads[..used_count] {
self.queue.add_used(&mem, desc_index, len);
}
used_count > 0
}
fn signal_used_queue(&self) -> result::Result<(), DeviceError> {
self.interrupt_cb
.trigger(&VirtioInterruptType::Queue, Some(&self.queue))
.map_err(|e| {
error!("Failed to signal used queue: {:?}", e);
DeviceError::FailedSignalingUsedQueue(e)
})
}
fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
// Create the epoll file descriptor
let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
// Use 'File' to enforce closing on 'epoll_fd'
let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
// Add events
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.queue_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(QUEUE_AVAIL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.kill_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.pause_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
const EPOLL_EVENTS_LEN: usize = 100;
let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
// Before jumping into the epoll loop, check if the device is expected
// to be in a paused state. This is helpful for the restore code path
// as the device thread should not start processing anything before the
// device has been resumed.
while paused.load(Ordering::SeqCst) {
thread::park();
}
'epoll: loop {
let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
Ok(res) => res,
Err(e) => {
if e.kind() == io::ErrorKind::Interrupted {
// It's well defined from the epoll_wait() syscall
// documentation that the epoll loop can be interrupted
// before any of the requested events occurred or the
// timeout expired. In both those cases, epoll_wait()
// returns an error of type EINTR, but this should not
// be considered as a regular error. Instead it is more
// appropriate to retry, by calling into epoll_wait().
continue;
}
return Err(DeviceError::EpollWait(e));
}
};
for event in events.iter().take(num_events) {
let ev_type = event.data as u16;
match ev_type {
QUEUE_AVAIL_EVENT => {
if let Err(e) = self.queue_evt.read() {
error!("Failed to get queue event: {:?}", e);
break 'epoll;
} else if self.process_queue() {
if let Err(e) = self.signal_used_queue() {
error!("Failed to signal used queue: {:?}", e);
break 'epoll;
}
}
}
KILL_EVENT => {
debug!("kill_evt received, stopping epoll loop");
break 'epoll;
}
PAUSE_EVENT => {
debug!("PAUSE_EVENT received, pausing virtio-pmem epoll loop");
// We loop here to handle spurious park() returns.
// Until we have not resumed, the paused boolean will
// be true.
while paused.load(Ordering::SeqCst) {
thread::park();
}
// Drain pause event after the device has been resumed.
// This ensures the pause event has been seen by each
// and every thread related to this virtio device.
let _ = self.pause_evt.read();
}
_ => {
error!("Unknown event for virtio-block");
}
}
}
}
Ok(())
}
}
pub struct Pmem {
id: String,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
disk: Option<File>,
avail_features: u64,
acked_features: u64,
config: VirtioPmemConfig,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
paused: Arc<AtomicBool>,
mapping: UserspaceMapping,
// Hold ownership of the memory that is allocated for the device
// which will be automatically dropped when the device is dropped
_region: MmapRegion,
}
#[derive(Serialize, Deserialize)]
pub struct PmemState {
avail_features: u64,
acked_features: u64,
config: VirtioPmemConfig,
}
impl Pmem {
pub fn new(
id: String,
disk: File,
addr: GuestAddress,
mapping: UserspaceMapping,
_region: MmapRegion,
iommu: bool,
) -> io::Result<Pmem> {
let config = VirtioPmemConfig {
start: addr.raw_value().to_le(),
size: (_region.size() as u64).to_le(),
};
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
if iommu {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
Ok(Pmem {
id,
kill_evt: None,
pause_evt: None,
disk: Some(disk),
avail_features,
acked_features: 0u64,
config,
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
paused: Arc::new(AtomicBool::new(false)),
mapping,
_region,
})
}
fn state(&self) -> PmemState {
PmemState {
avail_features: self.avail_features,
acked_features: self.acked_features,
config: self.config,
}
}
fn set_state(&mut self, state: &PmemState) -> io::Result<()> {
self.avail_features = state.avail_features;
self.acked_features = state.acked_features;
self.config = state.config;
Ok(())
}
}
impl Drop for Pmem {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
}
}
impl VirtioDevice for Pmem {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_PMEM as u32
}
fn queue_max_sizes(&self) -> &[u16] {
QUEUE_SIZES
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("Received acknowledge request for unknown feature.");
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, offset: u64, mut data: &mut [u8]) {
let config_slice = self.config.as_slice();
let config_len = config_slice.len() as u64;
if offset >= config_len {
error!("Failed to read config space");
return;
}
if let Some(end) = offset.checked_add(data.len() as u64) {
// This write can't fail, offset and end are checked against config_len.
data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
.unwrap();
}
}
fn write_config(&mut self, _offset: u64, _data: &[u8]) {
warn!("virtio-pmem device configuration is read-only");
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
mut queues: Vec<Queue>,
mut queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
NUM_QUEUES,
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
// Save the interrupt EventFD as we need to return it on reset
// but clone it to pass into the thread.
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
if let Some(disk) = self.disk.as_ref() {
let disk = disk.try_clone().map_err(|e| {
error!("failed cloning pmem disk: {}", e);
ActivateError::BadActivate
})?;
let mut handler = PmemEpollHandler {
queue: queues.remove(0),
mem,
disk,
interrupt_cb,
queue_evt: queue_evts.remove(0),
kill_evt,
pause_evt,
};
let paused = self.paused.clone();
let mut epoll_threads = Vec::new();
thread::Builder::new()
.name("virtio_pmem".to_string())
.spawn(move || handler.run(paused))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone virtio-pmem epoll thread: {}", e);
ActivateError::BadActivate
})?;
self.epoll_threads = Some(epoll_threads);
return Ok(());
}
Err(ActivateError::BadActivate)
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
fn userspace_mappings(&self) -> Vec<UserspaceMapping> {
vec![self.mapping.clone()]
}
}
virtio_pausable!(Pmem);
impl Snapshottable for Pmem {
fn id(&self) -> String {
self.id.clone()
}
fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
let snapshot =
serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
let mut pmem_snapshot = Snapshot::new(self.id.as_str());
pmem_snapshot.add_data_section(SnapshotDataSection {
id: format!("{}-section", self.id),
snapshot,
});
Ok(pmem_snapshot)
}
fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
if let Some(pmem_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
let pmem_state = match serde_json::from_slice(&pmem_section.snapshot) {
Ok(state) => state,
Err(error) => {
return Err(MigratableError::Restore(anyhow!(
"Could not deserialize PMEM {}",
error
)))
}
};
return self.set_state(&pmem_state).map_err(|e| {
MigratableError::Restore(anyhow!("Could not restore PMEM state {:?}", e))
});
}
Err(MigratableError::Restore(anyhow!(
"Could not find PMEM snapshot section"
)))
}
}
impl Transportable for Pmem {}
impl Migratable for Pmem {}

37
vm-virtio/src/queue.rs
View File

@@ -8,22 +8,21 @@
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
use crate::VirtioIommuRemapping;
use std::cmp::min;
use std::convert::TryInto;
use std::fmt::{self, Display};
use std::num::Wrapping;
use std::sync::atomic::{fence, Ordering};
use std::sync::Arc;
use crate::device::VirtioIommuRemapping;
use vm_memory::{
Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap,
GuestUsize,
};
pub(super) const VIRTQ_DESC_F_NEXT: u16 = 0x1;
pub(super) const VIRTQ_DESC_F_WRITE: u16 = 0x2;
pub(super) const VIRTQ_DESC_F_INDIRECT: u16 = 0x4;
pub const VIRTQ_DESC_F_NEXT: u16 = 0x1;
pub const VIRTQ_DESC_F_WRITE: u16 = 0x2;
pub const VIRTQ_DESC_F_INDIRECT: u16 = 0x4;
#[derive(Debug)]
pub enum Error {
@@ -718,15 +717,12 @@ impl Queue {
}
}
#[cfg(test)]
pub(crate) mod tests {
extern crate vm_memory;
pub mod testing {
use super::*;
use std::marker::PhantomData;
use std::mem;
pub use super::*;
use vm_memory::{GuestAddress, GuestMemoryMmap, GuestUsize};
use vm_memory::Bytes;
use vm_memory::{Address, GuestAddress, GuestMemoryMmap, GuestUsize};
// Represents a location in GuestMemoryMmap which holds a given type.
pub struct SomeplaceInMemory<'a, T> {
@@ -790,7 +786,7 @@ pub(crate) mod tests {
}
impl<'a> VirtqDesc<'a> {
fn new(start: GuestAddress, mem: &'a GuestMemoryMmap) -> Self {
pub fn new(start: GuestAddress, mem: &'a GuestMemoryMmap) -> Self {
assert_eq!(start.0 & 0xf, 0);
let addr = SomeplaceInMemory::new(start, mem);
@@ -930,15 +926,15 @@ pub(crate) mod tests {
self.dtable.len() as u16
}
fn dtable_start(&self) -> GuestAddress {
pub fn dtable_start(&self) -> GuestAddress {
self.dtable.first().unwrap().start()
}
fn avail_start(&self) -> GuestAddress {
pub fn avail_start(&self) -> GuestAddress {
self.avail.flags.location
}
fn used_start(&self) -> GuestAddress {
pub fn used_start(&self) -> GuestAddress {
self.used.flags.location
}
@@ -963,6 +959,15 @@ pub(crate) mod tests {
self.used.end()
}
}
}
#[cfg(test)]
pub mod tests {
extern crate vm_memory;
use super::testing::*;
pub use super::*;
use vm_memory::{GuestAddress, GuestMemoryMmap};
#[test]
fn test_checked_new_descriptor_chain() {

442
vm-virtio/src/rng.rs
View File

@@ -1,442 +0,0 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use super::Error as DeviceError;
use super::{
ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
};
use crate::{VirtioInterrupt, VirtioInterruptType};
use anyhow::anyhow;
use libc::EFD_NONBLOCK;
use std::fs::File;
use std::io;
use std::os::unix::io::{AsRawFd, FromRawFd};
use std::result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use vm_memory::{Bytes, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
use vm_migration::{
Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
Transportable,
};
use vmm_sys_util::eventfd::EventFd;
const QUEUE_SIZE: u16 = 256;
const NUM_QUEUES: usize = 1;
const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE];
// New descriptors are pending on the virtio queue.
const QUEUE_AVAIL_EVENT: DeviceEventT = 0;
// The device has been dropped.
const KILL_EVENT: DeviceEventT = 1;
// The device should be paused.
const PAUSE_EVENT: DeviceEventT = 2;
struct RngEpollHandler {
queues: Vec<Queue>,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
random_file: File,
interrupt_cb: Arc<dyn VirtioInterrupt>,
queue_evt: EventFd,
kill_evt: EventFd,
pause_evt: EventFd,
}
impl RngEpollHandler {
fn process_queue(&mut self) -> bool {
let queue = &mut self.queues[0];
let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
let mut used_count = 0;
let mem = self.mem.memory();
for avail_desc in queue.iter(&mem) {
let mut len = 0;
// Drivers can only read from the random device.
if avail_desc.is_write_only() {
// Fill the read with data from the random device on the host.
if mem
.read_from(
avail_desc.addr,
&mut self.random_file,
avail_desc.len as usize,
)
.is_ok()
{
len = avail_desc.len;
}
}
used_desc_heads[used_count] = (avail_desc.index, len);
used_count += 1;
}
for &(desc_index, len) in &used_desc_heads[..used_count] {
queue.add_used(&mem, desc_index, len);
}
used_count > 0
}
fn signal_used_queue(&self) -> result::Result<(), DeviceError> {
self.interrupt_cb
.trigger(&VirtioInterruptType::Queue, Some(&self.queues[0]))
.map_err(|e| {
error!("Failed to signal used queue: {:?}", e);
DeviceError::FailedSignalingUsedQueue(e)
})
}
fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
// Create the epoll file descriptor
let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
// Use 'File' to enforce closing on 'epoll_fd'
let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
// Add events
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.queue_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(QUEUE_AVAIL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.kill_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.pause_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
)
.map_err(DeviceError::EpollCtl)?;
const EPOLL_EVENTS_LEN: usize = 100;
let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
// Before jumping into the epoll loop, check if the device is expected
// to be in a paused state. This is helpful for the restore code path
// as the device thread should not start processing anything before the
// device has been resumed.
while paused.load(Ordering::SeqCst) {
thread::park();
}
'epoll: loop {
let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
Ok(res) => res,
Err(e) => {
if e.kind() == io::ErrorKind::Interrupted {
// It's well defined from the epoll_wait() syscall
// documentation that the epoll loop can be interrupted
// before any of the requested events occurred or the
// timeout expired. In both those cases, epoll_wait()
// returns an error of type EINTR, but this should not
// be considered as a regular error. Instead it is more
// appropriate to retry, by calling into epoll_wait().
continue;
}
return Err(DeviceError::EpollWait(e));
}
};
for event in events.iter().take(num_events) {
let ev_type = event.data as u16;
match ev_type {
QUEUE_AVAIL_EVENT => {
if let Err(e) = self.queue_evt.read() {
error!("Failed to get queue event: {:?}", e);
break 'epoll;
} else if self.process_queue() {
if let Err(e) = self.signal_used_queue() {
error!("Failed to signal used queue: {:?}", e);
break 'epoll;
}
}
}
KILL_EVENT => {
debug!("KILL_EVENT received, stopping epoll loop");
break 'epoll;
}
PAUSE_EVENT => {
debug!("PAUSE_EVENT received, pausing virtio-rng epoll loop");
// We loop here to handle spurious park() returns.
// Until we have not resumed, the paused boolean will
// be true.
while paused.load(Ordering::SeqCst) {
thread::park();
}
// Drain pause event after the device has been resumed.
// This ensures the pause event has been seen by each
// and every thread related to this virtio device.
let _ = self.pause_evt.read();
}
_ => {
error!("Unknown event for virtio-block");
}
}
}
}
Ok(())
}
}
/// Virtio device for exposing entropy to the guest OS through virtio.
pub struct Rng {
id: String,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
random_file: Option<File>,
avail_features: u64,
acked_features: u64,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
paused: Arc<AtomicBool>,
}
#[derive(Serialize, Deserialize)]
pub struct RngState {
pub avail_features: u64,
pub acked_features: u64,
pub paused: Arc<AtomicBool>,
}
impl Rng {
/// Create a new virtio rng device that gets random data from /dev/urandom.
pub fn new(id: String, path: &str, iommu: bool) -> io::Result<Rng> {
let random_file = File::open(path)?;
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
if iommu {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
Ok(Rng {
id,
kill_evt: None,
pause_evt: None,
random_file: Some(random_file),
avail_features,
acked_features: 0u64,
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
paused: Arc::new(AtomicBool::new(false)),
})
}
fn state(&self) -> RngState {
RngState {
avail_features: self.avail_features,
acked_features: self.acked_features,
paused: self.paused.clone(),
}
}
fn set_state(&mut self, state: &RngState) -> io::Result<()> {
self.avail_features = state.avail_features;
self.acked_features = state.acked_features;
self.paused = state.paused.clone();
Ok(())
}
}
impl Drop for Rng {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
}
}
impl VirtioDevice for Rng {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_RNG as u32
}
fn queue_max_sizes(&self) -> &[u16] {
QUEUE_SIZES
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("Received acknowledge request for unknown feature.");
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, _offset: u64, _data: &mut [u8]) {
warn!("No currently device specific configration defined");
}
fn write_config(&mut self, _offset: u64, _data: &[u8]) {
warn!("No currently device specific configration defined");
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
queues: Vec<Queue>,
mut queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
NUM_QUEUES,
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
// Save the interrupt EventFD as we need to return it on reset
// but clone it to pass into the thread.
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
if let Some(file) = self.random_file.as_ref() {
let random_file = file.try_clone().map_err(|e| {
error!("failed cloning rng source: {}", e);
ActivateError::BadActivate
})?;
let mut handler = RngEpollHandler {
queues,
mem,
random_file,
interrupt_cb,
queue_evt: queue_evts.remove(0),
kill_evt,
pause_evt,
};
let paused = self.paused.clone();
let mut epoll_threads = Vec::new();
thread::Builder::new()
.name("virtio_rng".to_string())
.spawn(move || handler.run(paused))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone the virtio-rng epoll thread: {}", e);
ActivateError::BadActivate
})?;
self.epoll_threads = Some(epoll_threads);
return Ok(());
}
Err(ActivateError::BadActivate)
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
// Then kill it.
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
}
virtio_pausable!(Rng);
impl Snapshottable for Rng {
fn id(&self) -> String {
self.id.clone()
}
fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
let snapshot =
serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
let mut rng_snapshot = Snapshot::new(self.id.as_str());
rng_snapshot.add_data_section(SnapshotDataSection {
id: format!("{}-section", self.id),
snapshot,
});
Ok(rng_snapshot)
}
fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
if let Some(rng_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
let rng_state = match serde_json::from_slice(&rng_section.snapshot) {
Ok(state) => state,
Err(error) => {
return Err(MigratableError::Restore(anyhow!(
"Could not deserialize RNG {}",
error
)))
}
};
return self.set_state(&rng_state).map_err(|e| {
MigratableError::Restore(anyhow!("Could not restore RNG state {:?}", e))
});
}
Err(MigratableError::Restore(anyhow!(
"Could not find RNG snapshot section"
)))
}
}
impl Transportable for Rng {}
impl Migratable for Rng {}

536
vm-virtio/src/transport/mmio.rs
View File

@@ -1,536 +0,0 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use crate::transport::{VirtioTransport, NOTIFY_REG_OFFSET};
use crate::{
Queue, VirtioDevice, VirtioInterrupt, VirtioInterruptType, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER,
DEVICE_DRIVER_OK, DEVICE_FAILED, DEVICE_FEATURES_OK, DEVICE_INIT,
INTERRUPT_STATUS_CONFIG_CHANGED, INTERRUPT_STATUS_USED_RING,
};
use anyhow::anyhow;
use byteorder::{ByteOrder, LittleEndian};
use devices::BusDevice;
use libc::EFD_NONBLOCK;
use std::num::Wrapping;
use std::result;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use vm_device::interrupt::InterruptSourceGroup;
use vm_memory::{GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
use vm_migration::{
Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
Transportable,
};
use vmm_sys_util::{errno::Result, eventfd::EventFd};
const VENDOR_ID: u32 = 0;
const MMIO_MAGIC_VALUE: u32 = 0x7472_6976;
const MMIO_VERSION: u32 = 2;
#[derive(Debug)]
enum Error {
/// Failed to retrieve queue ring's index.
QueueRingIndex(crate::queue::Error),
}
pub struct VirtioInterruptIntx {
interrupt_status: Arc<AtomicUsize>,
interrupt: Arc<Box<dyn InterruptSourceGroup>>,
}
impl VirtioInterruptIntx {
pub fn new(
interrupt_status: Arc<AtomicUsize>,
interrupt: Arc<Box<dyn InterruptSourceGroup>>,
) -> Self {
VirtioInterruptIntx {
interrupt_status,
interrupt,
}
}
}
impl VirtioInterrupt for VirtioInterruptIntx {
fn trigger(
&self,
int_type: &VirtioInterruptType,
_queue: Option<&Queue>,
) -> std::result::Result<(), std::io::Error> {
let status = match int_type {
VirtioInterruptType::Config => INTERRUPT_STATUS_CONFIG_CHANGED,
VirtioInterruptType::Queue => INTERRUPT_STATUS_USED_RING,
};
self.interrupt_status
.fetch_or(status as usize, Ordering::SeqCst);
self.interrupt.trigger(0)
}
}
#[derive(Serialize, Deserialize)]
struct VirtioMmioDeviceState {
device_activated: bool,
features_select: u32,
acked_features_select: u32,
queue_select: u32,
interrupt_status: usize,
driver_status: u32,
queues: Vec<Queue>,
shm_region_select: u32,
}
/// Implements the
/// [MMIO](http://docs.oasis-open.org/virtio/virtio/v1.0/cs04/virtio-v1.0-cs04.html#x1-1090002)
/// transport for virtio devices.
///
/// This requires 3 points of installation to work with a VM:
///
/// 1. Mmio reads and writes must be sent to this device at what is referred to here as MMIO base.
/// 1. `Mmio::queue_evts` must be installed at `virtio::NOTIFY_REG_OFFSET` offset from the MMIO
/// base. Each event in the array must be signaled if the index is written at that offset.
/// 1. `Mmio::interrupt_evt` must signal an interrupt that the guest driver is listening to when it
/// is written to.
///
/// Typically one page (4096 bytes) of MMIO address space is sufficient to handle this transport
/// and inner virtio device.
pub struct MmioDevice {
id: String,
device: Arc<Mutex<dyn VirtioDevice>>,
device_activated: bool,
features_select: u32,
acked_features_select: u32,
queue_select: u32,
interrupt_status: Arc<AtomicUsize>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
driver_status: u32,
config_generation: u32,
queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
mem: Option<GuestMemoryAtomic<GuestMemoryMmap>>,
shm_region_select: u32,
}
impl MmioDevice {
/// Constructs a new MMIO transport for the given virtio device.
pub fn new(
id: String,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
device: Arc<Mutex<dyn VirtioDevice>>,
) -> Result<MmioDevice> {
let device_clone = device.clone();
let locked_device = device_clone.lock().unwrap();
let mut queue_evts = Vec::new();
for _ in locked_device.queue_max_sizes().iter() {
queue_evts.push(EventFd::new(EFD_NONBLOCK)?)
}
let queues = locked_device
.queue_max_sizes()
.iter()
.map(|&s| Queue::new(s))
.collect();
Ok(MmioDevice {
id,
device,
device_activated: false,
features_select: 0,
acked_features_select: 0,
queue_select: 0,
interrupt_status: Arc::new(AtomicUsize::new(0)),
interrupt_cb: None,
driver_status: DEVICE_INIT,
config_generation: 0,
queues,
queue_evts,
mem: Some(mem),
shm_region_select: 0,
})
}
fn state(&self) -> VirtioMmioDeviceState {
VirtioMmioDeviceState {
device_activated: self.device_activated,
features_select: self.features_select,
acked_features_select: self.acked_features_select,
queue_select: self.queue_select,
interrupt_status: self.interrupt_status.load(Ordering::SeqCst),
driver_status: self.driver_status,
queues: self.queues.clone(),
shm_region_select: self.shm_region_select,
}
}
fn set_state(&mut self, state: &VirtioMmioDeviceState) -> std::result::Result<(), Error> {
self.device_activated = state.device_activated;
self.features_select = state.features_select;
self.acked_features_select = state.acked_features_select;
self.queue_select = state.queue_select;
self.interrupt_status
.store(state.interrupt_status, Ordering::SeqCst);
self.driver_status = state.driver_status;
// Update virtqueues indexes for both available and used rings.
if let Some(mem) = self.mem.as_ref() {
let mem = mem.memory();
for (i, queue) in self.queues.iter_mut().enumerate() {
queue.max_size = state.queues[i].max_size;
queue.size = state.queues[i].size;
queue.ready = state.queues[i].ready;
queue.vector = state.queues[i].vector;
queue.desc_table = state.queues[i].desc_table;
queue.avail_ring = state.queues[i].avail_ring;
queue.used_ring = state.queues[i].used_ring;
queue.next_avail = Wrapping(
queue
.used_index_from_memory(&mem)
.map_err(Error::QueueRingIndex)?,
);
queue.next_used = Wrapping(
queue
.used_index_from_memory(&mem)
.map_err(Error::QueueRingIndex)?,
);
}
}
self.shm_region_select = state.shm_region_select;
Ok(())
}
/// Gets the list of queue events that must be triggered whenever the VM writes to
/// `virtio::NOTIFY_REG_OFFSET` past the MMIO base. Each event must be triggered when the
/// value being written equals the index of the event in this list.
fn queue_evts(&self) -> &[EventFd] {
self.queue_evts.as_slice()
}
fn is_driver_ready(&self) -> bool {
let ready_bits = DEVICE_ACKNOWLEDGE | DEVICE_DRIVER | DEVICE_DRIVER_OK | DEVICE_FEATURES_OK;
self.driver_status == ready_bits && self.driver_status & DEVICE_FAILED == 0
}
/// Determines if the driver has requested the device (re)init / reset itself
fn is_driver_init(&self) -> bool {
self.driver_status == DEVICE_INIT
}
fn are_queues_valid(&self) -> bool {
if let Some(mem) = self.mem.as_ref() {
self.queues.iter().all(|q| q.is_valid(&mem.memory()))
} else {
false
}
}
fn with_queue<U, F>(&self, d: U, f: F) -> U
where
F: FnOnce(&Queue) -> U,
{
match self.queues.get(self.queue_select as usize) {
Some(queue) => f(queue),
None => d,
}
}
fn with_queue_mut<F: FnOnce(&mut Queue)>(&mut self, f: F) -> bool {
if let Some(queue) = self.queues.get_mut(self.queue_select as usize) {
f(queue);
true
} else {
false
}
}
pub fn assign_interrupt(&mut self, interrupt: Arc<Box<dyn InterruptSourceGroup>>) {
self.interrupt_cb = Some(Arc::new(VirtioInterruptIntx::new(
self.interrupt_status.clone(),
interrupt,
)));
}
}
impl VirtioTransport for MmioDevice {
fn ioeventfds(&self, base_addr: u64) -> Vec<(&EventFd, u64)> {
let notify_base = base_addr + u64::from(NOTIFY_REG_OFFSET);
self.queue_evts()
.iter()
.map(|event| (event, notify_base))
.collect()
}
}
impl BusDevice for MmioDevice {
fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
match offset {
0x00..=0xff if data.len() == 4 => {
let v = match offset {
0x0 => MMIO_MAGIC_VALUE,
0x04 => MMIO_VERSION,
0x08 => self.device.lock().unwrap().device_type(),
0x0c => VENDOR_ID, // vendor id
0x10 => {
if self.features_select < 2 {
(self.device.lock().unwrap().features() >> (self.features_select * 32))
as u32
} else {
0
}
}
0x34 => self.with_queue(0, |q| u32::from(q.get_max_size())),
0x44 => self.with_queue(0, |q| q.ready as u32),
0x60 => self.interrupt_status.load(Ordering::SeqCst) as u32,
0x70 => self.driver_status,
0xfc => self.config_generation,
0xb0..=0xbc => {
// For no SHM region or invalid region the kernel looks for length of -1
let (shm_offset, shm_len) = if let Some(shm_regions) =
self.device.lock().unwrap().get_shm_regions()
{
if self.shm_region_select as usize > shm_regions.region_list.len() {
(0, !0 as u64)
} else {
(
shm_regions.region_list[self.shm_region_select as usize].offset
+ shm_regions.addr.0,
shm_regions.region_list[self.shm_region_select as usize].len,
)
}
} else {
(0, !0 as u64)
};
match offset {
0xb0 => shm_len as u32,
0xb4 => (shm_len >> 32) as u32,
0xb8 => shm_offset as u32,
0xbc => (shm_offset >> 32) as u32,
_ => {
error!("invalid shm region offset");
0
}
}
}
_ => {
warn!("unknown virtio mmio register read: 0x{:x}", offset);
return;
}
};
LittleEndian::write_u32(data, v);
}
0x100..=0xfff => self
.device
.lock()
.unwrap()
.read_config(offset - 0x100, data),
_ => {
warn!(
"invalid virtio mmio read: 0x{:x}:0x{:x}",
offset,
data.len()
);
}
};
}
fn write(&mut self, _base: u64, offset: u64, data: &[u8]) {
fn hi(v: &mut GuestAddress, x: u32) {
*v = (*v & 0xffff_ffff) | (u64::from(x) << 32)
}
fn lo(v: &mut GuestAddress, x: u32) {
*v = (*v & !0xffff_ffff) | u64::from(x)
}
let mut mut_q = false;
match offset {
0x00..=0xff if data.len() == 4 => {
let v = LittleEndian::read_u32(data);
match offset {
0x14 => self.features_select = v,
0x20 => {
if self.acked_features_select < 2 {
self.device
.lock()
.unwrap()
.ack_features(u64::from(v) << (self.acked_features_select * 32));
} else {
warn!(
"invalid ack_features (page {}, value 0x{:x})",
self.acked_features_select, v
);
}
}
0x24 => self.acked_features_select = v,
0x30 => self.queue_select = v,
0x38 => mut_q = self.with_queue_mut(|q| q.size = v as u16),
0x44 => mut_q = self.with_queue_mut(|q| q.ready = v == 1),
0x64 => {
self.interrupt_status
.fetch_and(!(v as usize), Ordering::SeqCst);
}
0x70 => self.driver_status = v,
0x80 => mut_q = self.with_queue_mut(|q| lo(&mut q.desc_table, v)),
0x84 => mut_q = self.with_queue_mut(|q| hi(&mut q.desc_table, v)),
0x90 => mut_q = self.with_queue_mut(|q| lo(&mut q.avail_ring, v)),
0x94 => mut_q = self.with_queue_mut(|q| hi(&mut q.avail_ring, v)),
0xa0 => mut_q = self.with_queue_mut(|q| lo(&mut q.used_ring, v)),
0xa4 => mut_q = self.with_queue_mut(|q| hi(&mut q.used_ring, v)),
0xac => self.shm_region_select = v,
_ => {
warn!("unknown virtio mmio register write: 0x{:x}", offset);
return;
}
}
}
0x100..=0xfff => {
return self
.device
.lock()
.unwrap()
.write_config(offset - 0x100, data)
}
_ => {
warn!(
"invalid virtio mmio write: 0x{:x}:0x{:x}",
offset,
data.len()
);
return;
}
}
if self.device_activated && mut_q {
warn!("virtio queue was changed after device was activated");
}
if !self.device_activated && self.is_driver_ready() && self.are_queues_valid() {
if let Some(interrupt_cb) = self.interrupt_cb.take() {
if self.mem.is_some() {
let mem = self.mem.as_ref().unwrap().clone();
self.device
.lock()
.unwrap()
.activate(
mem,
interrupt_cb,
self.queues.clone(),
self.queue_evts.split_off(0),
)
.expect("Failed to activate device");
self.device_activated = true;
}
}
}
// Device has been reset by the driver
if self.device_activated && self.is_driver_init() {
let mut device = self.device.lock().unwrap();
if let Some((interrupt_cb, mut queue_evts)) = device.reset() {
// Upon reset the device returns its interrupt EventFD and it's queue EventFDs
self.interrupt_cb = Some(interrupt_cb);
self.queue_evts.append(&mut queue_evts);
self.device_activated = false;
// Reset queue readiness (changes queue_enable), queue sizes
// and selected_queue as per spec for reset
self.queues.iter_mut().for_each(Queue::reset);
self.queue_select = 0;
} else {
error!("Attempt to reset device when not implemented in underlying device");
self.driver_status = DEVICE_FAILED;
}
}
}
}
impl Pausable for MmioDevice {
fn pause(&mut self) -> result::Result<(), MigratableError> {
Ok(())
}
fn resume(&mut self) -> result::Result<(), MigratableError> {
Ok(())
}
}
impl Snapshottable for MmioDevice {
fn id(&self) -> String {
self.id.clone()
}
fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
let snapshot =
serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
let mut virtio_mmio_dev_snapshot = Snapshot::new(self.id.as_str());
virtio_mmio_dev_snapshot.add_data_section(SnapshotDataSection {
id: format!("{}-section", self.id),
snapshot,
});
Ok(virtio_mmio_dev_snapshot)
}
fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
if let Some(virtio_mmio_dev_section) =
snapshot.snapshot_data.get(&format!("{}-section", self.id))
{
let virtio_mmio_dev_state =
match serde_json::from_slice(&virtio_mmio_dev_section.snapshot) {
Ok(state) => state,
Err(error) => {
return Err(MigratableError::Restore(anyhow!(
"Could not deserialize VIRTIO_MMIO_DEVICE {}",
error
)))
}
};
// First restore the status of the virtqueues.
self.set_state(&virtio_mmio_dev_state).map_err(|e| {
MigratableError::Restore(anyhow!(
"Could not restore VIRTIO_MMIO_DEVICE state {:?}",
e
))
})?;
// Then we can activate the device, as we know at this point that
// the virtqueues are in the right state and the device is ready
// to be activated, which will spawn each virtio worker thread.
if self.device_activated && self.is_driver_ready() && self.are_queues_valid() {
if let Some(interrupt_cb) = self.interrupt_cb.take() {
if self.mem.is_some() {
let mem = self.mem.as_ref().unwrap().clone();
self.device
.lock()
.unwrap()
.activate(
mem,
interrupt_cb,
self.queues.clone(),
self.queue_evts.split_off(0),
)
.map_err(|e| {
MigratableError::Restore(anyhow!(
"Failed activating the device: {:?}",
e
))
})?;
}
}
}
return Ok(());
}
Err(MigratableError::Restore(anyhow!(
"Could not find VIRTIO_MMIO_DEVICE snapshot section"
)))
}
}
impl Transportable for MmioDevice {}
impl Migratable for MmioDevice {}

24
vm-virtio/src/transport/mod.rs
View File

@@ -1,24 +0,0 @@
// Copyright © 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
use vmm_sys_util::eventfd::EventFd;
#[cfg(feature = "pci_support")]
mod pci_common_config;
#[cfg(feature = "pci_support")]
mod pci_device;
#[cfg(feature = "pci_support")]
pub use pci_common_config::VirtioPciCommonConfig;
#[cfg(feature = "pci_support")]
pub use pci_device::VirtioPciDevice;
#[cfg(feature = "mmio_support")]
mod mmio;
#[cfg(feature = "mmio_support")]
pub use mmio::MmioDevice;
#[cfg(feature = "mmio_support")]
pub const NOTIFY_REG_OFFSET: u32 = 0x50;
pub trait VirtioTransport {
fn ioeventfds(&self, base_addr: u64) -> Vec<(&EventFd, u64)>;
}

423
vm-virtio/src/transport/pci_common_config.rs
View File

@@ -1,423 +0,0 @@
// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-BSD-3-Clause file.
//
// Copyright © 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
extern crate byteorder;
use crate::{Queue, VirtioDevice};
use anyhow::anyhow;
use byteorder::{ByteOrder, LittleEndian};
use std::sync::atomic::{AtomicU16, Ordering};
use std::sync::{Arc, Mutex};
use vm_memory::GuestAddress;
use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
#[derive(Clone, Serialize, Deserialize)]
pub struct VirtioPciCommonConfigState {
pub driver_status: u8,
pub config_generation: u8,
pub device_feature_select: u32,
pub driver_feature_select: u32,
pub queue_select: u16,
pub msix_config: u16,
}
/// Contains the data for reading and writing the common configuration structure of a virtio PCI
/// device.
///
/// * Registers:
/// ** About the whole device.
/// le32 device_feature_select; // 0x00 // read-write
/// le32 device_feature; // 0x04 // read-only for driver
/// le32 driver_feature_select; // 0x08 // read-write
/// le32 driver_feature; // 0x0C // read-write
/// le16 msix_config; // 0x10 // read-write
/// le16 num_queues; // 0x12 // read-only for driver
/// u8 device_status; // 0x14 // read-write (driver_status)
/// u8 config_generation; // 0x15 // read-only for driver
/// ** About a specific virtqueue.
/// le16 queue_select; // 0x16 // read-write
/// le16 queue_size; // 0x18 // read-write, power of 2, or 0.
/// le16 queue_msix_vector; // 0x1A // read-write
/// le16 queue_enable; // 0x1C // read-write (Ready)
/// le16 queue_notify_off; // 0x1E // read-only for driver
/// le64 queue_desc; // 0x20 // read-write
/// le64 queue_avail; // 0x28 // read-write
/// le64 queue_used; // 0x30 // read-write
pub struct VirtioPciCommonConfig {
pub driver_status: u8,
pub config_generation: u8,
pub device_feature_select: u32,
pub driver_feature_select: u32,
pub queue_select: u16,
pub msix_config: Arc<AtomicU16>,
}
impl VirtioPciCommonConfig {
fn state(&self) -> VirtioPciCommonConfigState {
VirtioPciCommonConfigState {
driver_status: self.driver_status,
config_generation: self.config_generation,
device_feature_select: self.device_feature_select,
driver_feature_select: self.driver_feature_select,
queue_select: self.queue_select,
msix_config: self.msix_config.load(Ordering::SeqCst),
}
}
fn set_state(&mut self, state: &VirtioPciCommonConfigState) {
self.driver_status = state.driver_status;
self.config_generation = state.config_generation;
self.device_feature_select = state.device_feature_select;
self.driver_feature_select = state.driver_feature_select;
self.queue_select = state.queue_select;
self.msix_config.store(state.msix_config, Ordering::SeqCst);
}
pub fn read(
&mut self,
offset: u64,
data: &mut [u8],
queues: &mut Vec<Queue>,
device: Arc<Mutex<dyn VirtioDevice>>,
) {
assert!(data.len() <= 8);
match data.len() {
1 => {
let v = self.read_common_config_byte(offset);
data[0] = v;
}
2 => {
let v = self.read_common_config_word(offset, queues);
LittleEndian::write_u16(data, v);
}
4 => {
let v = self.read_common_config_dword(offset, device);
LittleEndian::write_u32(data, v);
}
8 => {
let v = self.read_common_config_qword(offset);
LittleEndian::write_u64(data, v);
}
_ => error!("invalid data length for virtio read: len {}", data.len()),
}
}
pub fn write(
&mut self,
offset: u64,
data: &[u8],
queues: &mut Vec<Queue>,
device: Arc<Mutex<dyn VirtioDevice>>,
) {
assert!(data.len() <= 8);
match data.len() {
1 => self.write_common_config_byte(offset, data[0]),
2 => self.write_common_config_word(offset, LittleEndian::read_u16(data), queues),
4 => {
self.write_common_config_dword(offset, LittleEndian::read_u32(data), queues, device)
}
8 => self.write_common_config_qword(offset, LittleEndian::read_u64(data), queues),
_ => error!("invalid data length for virtio write: len {}", data.len()),
}
}
fn read_common_config_byte(&self, offset: u64) -> u8 {
debug!("read_common_config_byte: offset 0x{:x}", offset);
// The driver is only allowed to do aligned, properly sized access.
match offset {
0x14 => self.driver_status,
0x15 => self.config_generation,
_ => {
warn!("invalid virtio config byte read: 0x{:x}", offset);
0
}
}
}
fn write_common_config_byte(&mut self, offset: u64, value: u8) {
debug!("write_common_config_byte: offset 0x{:x}", offset);
match offset {
0x14 => self.driver_status = value,
_ => {
warn!("invalid virtio config byte write: 0x{:x}", offset);
}
}
}
fn read_common_config_word(&self, offset: u64, queues: &[Queue]) -> u16 {
debug!("read_common_config_word: offset 0x{:x}", offset);
match offset {
0x10 => self.msix_config.load(Ordering::SeqCst),
0x12 => queues.len() as u16, // num_queues
0x16 => self.queue_select,
0x18 => self.with_queue(queues, |q| q.size).unwrap_or(0),
0x1a => self.with_queue(queues, |q| q.vector).unwrap_or(0),
0x1c => {
if self.with_queue(queues, |q| q.ready).unwrap_or(false) {
1
} else {
0
}
}
0x1e => self.queue_select, // notify_off
_ => {
warn!("invalid virtio register word read: 0x{:x}", offset);
0
}
}
}
fn write_common_config_word(&mut self, offset: u64, value: u16, queues: &mut Vec<Queue>) {
debug!("write_common_config_word: offset 0x{:x}", offset);
match offset {
0x10 => self.msix_config.store(value, Ordering::SeqCst),
0x16 => self.queue_select = value,
0x18 => self.with_queue_mut(queues, |q| q.size = value),
0x1a => self.with_queue_mut(queues, |q| q.vector = value),
0x1c => self.with_queue_mut(queues, |q| q.enable(value == 1)),
_ => {
warn!("invalid virtio register word write: 0x{:x}", offset);
}
}
}
fn read_common_config_dword(&self, offset: u64, device: Arc<Mutex<dyn VirtioDevice>>) -> u32 {
debug!("read_common_config_dword: offset 0x{:x}", offset);
match offset {
0x00 => self.device_feature_select,
0x04 => {
let locked_device = device.lock().unwrap();
// Only 64 bits of features (2 pages) are defined for now, so limit
// device_feature_select to avoid shifting by 64 or more bits.
if self.device_feature_select < 2 {
(locked_device.features() >> (self.device_feature_select * 32)) as u32
} else {
0
}
}
0x08 => self.driver_feature_select,
_ => {
warn!("invalid virtio register dword read: 0x{:x}", offset);
0
}
}
}
fn write_common_config_dword(
&mut self,
offset: u64,
value: u32,
queues: &mut Vec<Queue>,
device: Arc<Mutex<dyn VirtioDevice>>,
) {
debug!("write_common_config_dword: offset 0x{:x}", offset);
fn hi(v: &mut GuestAddress, x: u32) {
*v = (*v & 0xffff_ffff) | ((u64::from(x)) << 32)
}
fn lo(v: &mut GuestAddress, x: u32) {
*v = (*v & !0xffff_ffff) | (u64::from(x))
}
match offset {
0x00 => self.device_feature_select = value,
0x08 => self.driver_feature_select = value,
0x0c => {
if self.driver_feature_select < 2 {
let mut locked_device = device.lock().unwrap();
locked_device
.ack_features(u64::from(value) << (self.driver_feature_select * 32));
} else {
warn!(
"invalid ack_features (page {}, value 0x{:x})",
self.driver_feature_select, value
);
}
}
0x20 => self.with_queue_mut(queues, |q| lo(&mut q.desc_table, value)),
0x24 => self.with_queue_mut(queues, |q| hi(&mut q.desc_table, value)),
0x28 => self.with_queue_mut(queues, |q| lo(&mut q.avail_ring, value)),
0x2c => self.with_queue_mut(queues, |q| hi(&mut q.avail_ring, value)),
0x30 => self.with_queue_mut(queues, |q| lo(&mut q.used_ring, value)),
0x34 => self.with_queue_mut(queues, |q| hi(&mut q.used_ring, value)),
_ => {
warn!("invalid virtio register dword write: 0x{:x}", offset);
}
}
}
fn read_common_config_qword(&self, _offset: u64) -> u64 {
debug!("read_common_config_qword: offset 0x{:x}", _offset);
0 // Assume the guest has no reason to read write-only registers.
}
fn write_common_config_qword(&mut self, offset: u64, value: u64, queues: &mut Vec<Queue>) {
debug!("write_common_config_qword: offset 0x{:x}", offset);
match offset {
0x20 => self.with_queue_mut(queues, |q| q.desc_table = GuestAddress(value)),
0x28 => self.with_queue_mut(queues, |q| q.avail_ring = GuestAddress(value)),
0x30 => self.with_queue_mut(queues, |q| q.used_ring = GuestAddress(value)),
_ => {
warn!("invalid virtio register qword write: 0x{:x}", offset);
}
}
}
fn with_queue<U, F>(&self, queues: &[Queue], f: F) -> Option<U>
where
F: FnOnce(&Queue) -> U,
{
queues.get(self.queue_select as usize).map(f)
}
fn with_queue_mut<F: FnOnce(&mut Queue)>(&self, queues: &mut Vec<Queue>, f: F) {
if let Some(queue) = queues.get_mut(self.queue_select as usize) {
f(queue);
}
}
}
impl Pausable for VirtioPciCommonConfig {}
impl Snapshottable for VirtioPciCommonConfig {
fn id(&self) -> String {
String::from("virtio_pci_common_config")
}
fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
let snapshot =
serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
let mut config_snapshot = Snapshot::new(self.id().as_str());
config_snapshot.add_data_section(SnapshotDataSection {
id: format!("{}-section", self.id()),
snapshot,
});
Ok(config_snapshot)
}
fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
if let Some(config_section) = snapshot
.snapshot_data
.get(&format!("{}-section", self.id()))
{
let config_state = match serde_json::from_slice(&config_section.snapshot) {
Ok(state) => state,
Err(error) => {
return Err(MigratableError::Restore(anyhow!(
"Could not deserialize {}: {}",
self.id(),
error
)))
}
};
self.set_state(&config_state);
return Ok(());
}
Err(MigratableError::Restore(anyhow!(
"Could not find {} snapshot section",
self.id()
)))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{ActivateResult, VirtioInterrupt};
use std::sync::Arc;
use vm_memory::{GuestMemoryAtomic, GuestMemoryMmap};
use vmm_sys_util::eventfd::EventFd;
struct DummyDevice(u32);
const QUEUE_SIZE: u16 = 256;
const QUEUE_SIZES: &'static [u16] = &[QUEUE_SIZE];
const DUMMY_FEATURES: u64 = 0x5555_aaaa;
impl VirtioDevice for DummyDevice {
fn device_type(&self) -> u32 {
return self.0;
}
fn queue_max_sizes(&self) -> &[u16] {
QUEUE_SIZES
}
fn activate(
&mut self,
_mem: GuestMemoryAtomic<GuestMemoryMmap>,
_interrupt_evt: Arc<dyn VirtioInterrupt>,
_queues: Vec<Queue>,
_queue_evts: Vec<EventFd>,
) -> ActivateResult {
Ok(())
}
fn features(&self) -> u64 {
DUMMY_FEATURES
}
fn ack_features(&mut self, _value: u64) {}
fn read_config(&self, _offset: u64, _data: &mut [u8]) {}
fn write_config(&mut self, _offset: u64, _data: &[u8]) {}
}
#[test]
fn write_base_regs() {
let mut regs = VirtioPciCommonConfig {
driver_status: 0xaa,
config_generation: 0x55,
device_feature_select: 0x0,
driver_feature_select: 0x0,
queue_select: 0xff,
msix_config: Arc::new(AtomicU16::new(0)),
};
let dev = Arc::new(Mutex::new(DummyDevice(0)));
let mut queues = Vec::new();
// Can set all bits of driver_status.
regs.write(0x14, &[0x55], &mut queues, dev.clone());
let mut read_back = vec![0x00];
regs.read(0x14, &mut read_back, &mut queues, dev.clone());
assert_eq!(read_back[0], 0x55);
// The config generation register is read only.
regs.write(0x15, &[0xaa], &mut queues, dev.clone());
let mut read_back = vec![0x00];
regs.read(0x15, &mut read_back, &mut queues, dev.clone());
assert_eq!(read_back[0], 0x55);
// Device features is read-only and passed through from the device.
regs.write(0x04, &[0, 0, 0, 0], &mut queues, dev.clone());
let mut read_back = vec![0, 0, 0, 0];
regs.read(0x04, &mut read_back, &mut queues, dev.clone());
assert_eq!(LittleEndian::read_u32(&read_back), DUMMY_FEATURES as u32);
// Feature select registers are read/write.
regs.write(0x00, &[1, 2, 3, 4], &mut queues, dev.clone());
let mut read_back = vec![0, 0, 0, 0];
regs.read(0x00, &mut read_back, &mut queues, dev.clone());
assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
regs.write(0x08, &[1, 2, 3, 4], &mut queues, dev.clone());
let mut read_back = vec![0, 0, 0, 0];
regs.read(0x08, &mut read_back, &mut queues, dev.clone());
assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
// 'queue_select' can be read and written.
regs.write(0x16, &[0xaa, 0x55], &mut queues, dev.clone());
let mut read_back = vec![0x00, 0x00];
regs.read(0x16, &mut read_back, &mut queues, dev.clone());
assert_eq!(read_back[0], 0xaa);
assert_eq!(read_back[1], 0x55);
}
}

1143
vm-virtio/src/transport/pci_device.rs
View File

File diff suppressed because it is too large Load Diff

335
vm-virtio/src/vhost_user/blk.rs
View File

@@ -1,335 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
use super::handler::*;
use super::vu_common_ctrl::*;
use super::Error as DeviceError;
use super::{Error, Result};
use crate::block::VirtioBlockConfig;
use crate::VirtioInterrupt;
use libc::EFD_NONBLOCK;
use std::cmp;
use std::io::Write;
use std::mem;
use std::os::unix::io::AsRawFd;
use std::result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use std::vec::Vec;
use vhost_rs::vhost_user::message::VhostUserConfigFlags;
use vhost_rs::vhost_user::message::VHOST_USER_CONFIG_OFFSET;
use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
use vhost_rs::VhostBackend;
use virtio_bindings::bindings::virtio_blk::*;
use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
use vmm_sys_util::eventfd::EventFd;
struct SlaveReqHandler {}
impl VhostUserMasterReqHandler for SlaveReqHandler {}
pub struct Blk {
id: String,
vhost_user_blk: Master,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
avail_features: u64,
acked_features: u64,
config: VirtioBlockConfig,
queue_sizes: Vec<u16>,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
paused: Arc<AtomicBool>,
}
impl Blk {
/// Create a new vhost-user-blk device
pub fn new(id: String, vu_cfg: VhostUserConfig) -> Result<Blk> {
let mut vhost_user_blk = Master::connect(&vu_cfg.socket, vu_cfg.num_queues as u64)
.map_err(Error::VhostUserCreateMaster)?;
// Filling device and vring features VMM supports.
let mut avail_features = 1 << VIRTIO_BLK_F_SEG_MAX
| 1 << VIRTIO_BLK_F_RO
| 1 << VIRTIO_BLK_F_BLK_SIZE
| 1 << VIRTIO_BLK_F_FLUSH
| 1 << VIRTIO_BLK_F_TOPOLOGY
| 1 << VIRTIO_RING_F_EVENT_IDX
| 1 << VIRTIO_BLK_F_CONFIG_WCE
| 1 << VIRTIO_F_VERSION_1
| VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
if vu_cfg.num_queues > 1 {
avail_features |= 1 << VIRTIO_BLK_F_MQ;
}
// Set vhost-user owner.
vhost_user_blk
.set_owner()
.map_err(Error::VhostUserSetOwner)?;
// Get features from backend, do negotiation to get a feature collection which
// both VMM and backend support.
let backend_features = vhost_user_blk
.get_features()
.map_err(Error::VhostUserGetFeatures)?;
avail_features &= backend_features;
// Set features back is required by the vhost crate mechanism, since the
// later vhost call will check if features is filled in master before execution.
vhost_user_blk
.set_features(avail_features)
.map_err(Error::VhostUserSetFeatures)?;
// Identify if protocol features are supported by the slave.
let mut acked_features = 0;
if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
let mut protocol_features = vhost_user_blk
.get_protocol_features()
.map_err(Error::VhostUserGetProtocolFeatures)?;
protocol_features |= VhostUserProtocolFeatures::MQ;
protocol_features &= !VhostUserProtocolFeatures::INFLIGHT_SHMFD;
vhost_user_blk
.set_protocol_features(protocol_features)
.map_err(Error::VhostUserSetProtocolFeatures)?;
}
// Get the max queues number from backend, and the queue number set
// should be less than this max queue number.
let max_queues_num = vhost_user_blk
.get_queue_num()
.map_err(Error::VhostUserGetQueueMaxNum)?;
if vu_cfg.num_queues > max_queues_num as usize {
error!("vhost-user-blk has queue number: {} larger than the max queue number: {} backend allowed\n",
vu_cfg.num_queues, max_queues_num);
return Err(Error::BadQueueNum);
}
let config_len = mem::size_of::<VirtioBlockConfig>();
let config_space: Vec<u8> = vec![0u8; config_len as usize];
let (_, config_space) = vhost_user_blk
.get_config(
VHOST_USER_CONFIG_OFFSET,
config_len as u32,
VhostUserConfigFlags::WRITABLE,
config_space.as_slice(),
)
.unwrap();
let mut config = VirtioBlockConfig::default();
if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice()) {
config = *backend_config;
// Only set num_queues value(u16).
config.num_queues = vu_cfg.num_queues as u16;
}
// Send set_vring_base here, since it could tell backends, like SPDK,
// how many virt queues to be handled, which backend required to know
// at early stage.
for i in 0..vu_cfg.num_queues {
vhost_user_blk
.set_vring_base(i, 0)
.map_err(Error::VhostUserSetVringBase)?;
}
Ok(Blk {
id,
vhost_user_blk,
kill_evt: None,
pause_evt: None,
avail_features,
acked_features,
config,
queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues],
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
paused: Arc::new(AtomicBool::new(false)),
})
}
}
impl Drop for Blk {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
if let Err(e) = kill_evt.write(1) {
error!("failed to kill vhost-user-blk: {:?}", e);
}
}
}
}
impl VirtioDevice for Blk {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_BLOCK as u32
}
fn queue_max_sizes(&self) -> &[u16] {
&self.queue_sizes
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("Received acknowledge request for unknown feature: {:x}", v);
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, offset: u64, mut data: &mut [u8]) {
let config_slice = self.config.as_slice();
let config_len = config_slice.len() as u64;
if offset >= config_len {
error!("Failed to read config space");
return;
}
if let Some(end) = offset.checked_add(data.len() as u64) {
// This write can't fail, offset and end are checked against config_len.
data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
.unwrap();
}
}
fn write_config(&mut self, offset: u64, data: &[u8]) {
let config_slice = self.config.as_mut_slice();
let data_len = data.len() as u64;
let config_len = config_slice.len() as u64;
if offset + data_len > config_len {
error!("Failed to write config space");
return;
}
self.vhost_user_blk
.set_config(offset as u32, VhostUserConfigFlags::WRITABLE, data)
.expect("Failed to set config");
let (_, right) = config_slice.split_at_mut(offset as usize);
right.copy_from_slice(&data[..]);
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
) -> ActivateResult {
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
// Save the interrupt EventFD as we need to return it on reset
// but clone it to pass into the thread.
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
let mut vu_interrupt_list = setup_vhost_user(
&mut self.vhost_user_blk,
&mem.memory(),
queues,
queue_evts,
&interrupt_cb,
self.acked_features,
)
.map_err(ActivateError::VhostUserBlkSetup)?;
let mut epoll_threads = Vec::new();
for _ in 0..vu_interrupt_list.len() {
let mut interrupt_list_sub: Vec<(Option<EventFd>, Queue)> = Vec::with_capacity(1);
interrupt_list_sub.push(vu_interrupt_list.remove(0));
let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
interrupt_cb: interrupt_cb.clone(),
kill_evt: kill_evt.try_clone().unwrap(),
pause_evt: pause_evt.try_clone().unwrap(),
vu_interrupt_list: interrupt_list_sub,
slave_req_handler: None,
});
let paused = self.paused.clone();
thread::Builder::new()
.name("vhost_user_blk".to_string())
.spawn(move || handler.run(paused))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone virtio epoll thread: {}", e);
ActivateError::BadActivate
})?;
}
self.epoll_threads = Some(epoll_threads);
Ok(())
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
if let Err(e) = reset_vhost_user(&mut self.vhost_user_blk, self.queue_sizes.len()) {
error!("Failed to reset vhost-user daemon: {:?}", e);
return None;
}
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
fn shutdown(&mut self) {
let _ = unsafe { libc::close(self.vhost_user_blk.as_raw_fd()) };
}
fn update_memory(&mut self, mem: &GuestMemoryMmap) -> std::result::Result<(), crate::Error> {
update_mem_table(&mut self.vhost_user_blk, mem).map_err(crate::Error::VhostUserUpdateMemory)
}
}
virtio_pausable!(Blk);
impl Snapshottable for Blk {
fn id(&self) -> String {
self.id.clone()
}
}
impl Transportable for Blk {}
impl Migratable for Blk {}

617
vm-virtio/src/vhost_user/fs.rs
View File

@@ -1,617 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
use super::vu_common_ctrl::{reset_vhost_user, setup_vhost_user, update_mem_table};
use super::Error as DeviceError;
use super::{Error, Result};
use crate::vhost_user::handler::{VhostUserEpollConfig, VhostUserEpollHandler};
use crate::{
ActivateError, ActivateResult, Queue, UserspaceMapping, VirtioDevice, VirtioDeviceType,
VirtioInterrupt, VirtioSharedMemoryList, VIRTIO_F_VERSION_1,
};
use libc::{self, c_void, off64_t, pread64, pwrite64, EFD_NONBLOCK};
use std::cmp;
use std::io;
use std::io::Write;
use std::os::unix::io::{AsRawFd, RawFd};
use std::result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use vhost_rs::vhost_user::message::{
VhostUserFSSlaveMsg, VhostUserFSSlaveMsgFlags, VhostUserProtocolFeatures,
VhostUserVirtioFeatures, VHOST_USER_FS_SLAVE_ENTRIES,
};
use vhost_rs::vhost_user::{
HandlerResult, Master, MasterReqHandler, VhostUserMaster, VhostUserMasterReqHandler,
};
use vhost_rs::VhostBackend;
use vm_memory::{
Address, ByteValued, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic,
GuestMemoryMmap, MmapRegion,
};
use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
use vmm_sys_util::eventfd::EventFd;
const NUM_QUEUE_OFFSET: usize = 1;
struct SlaveReqHandler {
cache_offset: GuestAddress,
cache_size: u64,
mmap_cache_addr: u64,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
}
impl SlaveReqHandler {
// Make sure request is within cache range
fn is_req_valid(&self, offset: u64, len: u64) -> bool {
let end = match offset.checked_add(len) {
Some(n) => n,
None => return false,
};
!(offset >= self.cache_size || end > self.cache_size)
}
}
impl VhostUserMasterReqHandler for SlaveReqHandler {
fn handle_config_change(&mut self) -> HandlerResult<u64> {
debug!("handle_config_change");
Ok(0)
}
fn fs_slave_map(&mut self, fs: &VhostUserFSSlaveMsg, fd: RawFd) -> HandlerResult<u64> {
debug!("fs_slave_map");
for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
let offset = fs.cache_offset[i];
let len = fs.len[i];
// Ignore if the length is 0.
if len == 0 {
continue;
}
if !self.is_req_valid(offset, len) {
return Err(io::Error::from_raw_os_error(libc::EINVAL));
}
let addr = self.mmap_cache_addr + offset;
let ret = unsafe {
libc::mmap(
addr as *mut libc::c_void,
len as usize,
fs.flags[i].bits() as i32,
libc::MAP_SHARED | libc::MAP_FIXED,
fd,
fs.fd_offset[i] as libc::off_t,
)
};
if ret == libc::MAP_FAILED {
return Err(io::Error::last_os_error());
}
let ret = unsafe { libc::close(fd) };
if ret == -1 {
return Err(io::Error::last_os_error());
}
}
Ok(0)
}
fn fs_slave_unmap(&mut self, fs: &VhostUserFSSlaveMsg) -> HandlerResult<u64> {
debug!("fs_slave_unmap");
for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
let offset = fs.cache_offset[i];
let mut len = fs.len[i];
// Ignore if the length is 0.
if len == 0 {
continue;
}
// Need to handle a special case where the slave ask for the unmapping
// of the entire mapping.
if len == 0xffff_ffff_ffff_ffff {
len = self.cache_size;
}
if !self.is_req_valid(offset, len) {
return Err(io::Error::from_raw_os_error(libc::EINVAL));
}
let addr = self.mmap_cache_addr + offset;
let ret = unsafe {
libc::mmap(
addr as *mut libc::c_void,
len as usize,
libc::PROT_NONE,
libc::MAP_ANONYMOUS | libc::MAP_PRIVATE | libc::MAP_FIXED,
-1,
0 as libc::off_t,
)
};
if ret == libc::MAP_FAILED {
return Err(io::Error::last_os_error());
}
}
Ok(0)
}
fn fs_slave_sync(&mut self, fs: &VhostUserFSSlaveMsg) -> HandlerResult<u64> {
debug!("fs_slave_sync");
for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
let offset = fs.cache_offset[i];
let len = fs.len[i];
// Ignore if the length is 0.
if len == 0 {
continue;
}
if !self.is_req_valid(offset, len) {
return Err(io::Error::from_raw_os_error(libc::EINVAL));
}
let addr = self.mmap_cache_addr + offset;
let ret =
unsafe { libc::msync(addr as *mut libc::c_void, len as usize, libc::MS_SYNC) };
if ret == -1 {
return Err(io::Error::last_os_error());
}
}
Ok(0)
}
fn fs_slave_io(&mut self, fs: &VhostUserFSSlaveMsg, fd: RawFd) -> HandlerResult<u64> {
debug!("fs_slave_io");
let mut done: u64 = 0;
for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
// Ignore if the length is 0.
if fs.len[i] == 0 {
continue;
}
let mut foffset = fs.fd_offset[i];
let mut len = fs.len[i] as usize;
let gpa = fs.cache_offset[i];
let cache_end = self.cache_offset.raw_value() + self.cache_size;
let efault = libc::EFAULT;
let mut ptr = if gpa >= self.cache_offset.raw_value() && gpa < cache_end {
let offset = gpa
.checked_sub(self.cache_offset.raw_value())
.ok_or_else(|| io::Error::from_raw_os_error(efault))?;
let end = gpa
.checked_add(fs.len[i])
.ok_or_else(|| io::Error::from_raw_os_error(efault))?;
if end >= cache_end {
return Err(io::Error::from_raw_os_error(efault));
}
self.mmap_cache_addr + offset
} else {
self.mem
.memory()
.get_host_address(GuestAddress(gpa))
.map_err(|e| {
error!(
"Failed to find RAM region associated with guest physical address 0x{:x}: {:?}",
gpa, e
);
io::Error::from_raw_os_error(efault)
})? as u64
};
while len > 0 {
let ret = if (fs.flags[i] & VhostUserFSSlaveMsgFlags::MAP_W)
== VhostUserFSSlaveMsgFlags::MAP_W
{
debug!("write: foffset={}, len={}", foffset, len);
unsafe { pwrite64(fd, ptr as *const c_void, len as usize, foffset as off64_t) }
} else {
debug!("read: foffset={}, len={}", foffset, len);
unsafe { pread64(fd, ptr as *mut c_void, len as usize, foffset as off64_t) }
};
if ret < 0 {
return Err(io::Error::last_os_error());
}
if ret == 0 {
// EOF
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"failed to access whole buffer",
));
}
len -= ret as usize;
foffset += ret as u64;
ptr += ret as u64;
done += ret as u64;
}
}
let ret = unsafe { libc::close(fd) };
if ret == -1 {
return Err(io::Error::last_os_error());
}
Ok(done)
}
}
#[derive(Copy, Clone)]
#[repr(C, packed)]
struct VirtioFsConfig {
tag: [u8; 36],
num_request_queues: u32,
}
impl Default for VirtioFsConfig {
fn default() -> Self {
VirtioFsConfig {
tag: [0; 36],
num_request_queues: 0,
}
}
}
unsafe impl ByteValued for VirtioFsConfig {}
pub struct Fs {
id: String,
vu: Master,
queue_sizes: Vec<u16>,
avail_features: u64,
acked_features: u64,
config: VirtioFsConfig,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
// Hold ownership of the memory that is allocated for the device
// which will be automatically dropped when the device is dropped
cache: Option<(VirtioSharedMemoryList, MmapRegion)>,
slave_req_support: bool,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
paused: Arc<AtomicBool>,
}
impl Fs {
/// Create a new virtio-fs device.
pub fn new(
id: String,
path: &str,
tag: &str,
req_num_queues: usize,
queue_size: u16,
cache: Option<(VirtioSharedMemoryList, MmapRegion)>,
) -> Result<Fs> {
let mut slave_req_support = false;
// Calculate the actual number of queues needed.
let num_queues = NUM_QUEUE_OFFSET + req_num_queues;
// Connect to the vhost-user socket.
let mut master =
Master::connect(path, num_queues as u64).map_err(Error::VhostUserCreateMaster)?;
// Filling device and vring features VMM supports.
let mut avail_features =
1 << VIRTIO_F_VERSION_1 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
// Set vhost-user owner.
master.set_owner().map_err(Error::VhostUserSetOwner)?;
// Get features from backend, do negotiation to get a feature collection which
// both VMM and backend support.
let backend_features = master.get_features().map_err(Error::VhostUserGetFeatures)?;
avail_features &= backend_features;
// Set features back is required by the vhost crate mechanism, since the
// later vhost call will check if features is filled in master before execution.
master
.set_features(avail_features)
.map_err(Error::VhostUserSetFeatures)?;
// Identify if protocol features are supported by the slave.
let mut acked_features = 0;
if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
let mut protocol_features = master
.get_protocol_features()
.map_err(Error::VhostUserGetProtocolFeatures)?;
if cache.is_some() {
protocol_features &= VhostUserProtocolFeatures::MQ
| VhostUserProtocolFeatures::REPLY_ACK
| VhostUserProtocolFeatures::SLAVE_REQ
| VhostUserProtocolFeatures::SLAVE_SEND_FD;
} else {
protocol_features &=
VhostUserProtocolFeatures::MQ | VhostUserProtocolFeatures::REPLY_ACK;
}
master
.set_protocol_features(protocol_features)
.map_err(Error::VhostUserSetProtocolFeatures)?;
slave_req_support = true;
}
// Create virtio-fs device configuration.
let mut config = VirtioFsConfig::default();
let tag_bytes_vec = tag.to_string().into_bytes();
config.tag[..tag_bytes_vec.len()].copy_from_slice(tag_bytes_vec.as_slice());
config.num_request_queues = req_num_queues as u32;
Ok(Fs {
id,
vu: master,
queue_sizes: vec![queue_size; num_queues],
avail_features,
acked_features,
config,
kill_evt: None,
pause_evt: None,
cache,
slave_req_support,
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
paused: Arc::new(AtomicBool::new(false)),
})
}
}
impl Drop for Fs {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
}
}
impl VirtioDevice for Fs {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_FS as u32
}
fn queue_max_sizes(&self) -> &[u16] {
&self.queue_sizes.as_slice()
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("fs: virtio-fs got unknown feature ack: {:x}", v);
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, offset: u64, mut data: &mut [u8]) {
let config_slice = self.config.as_slice();
let config_len = config_slice.len() as u64;
if offset >= config_len {
error!("Failed to read config space");
return;
}
if let Some(end) = offset.checked_add(data.len() as u64) {
// This write can't fail, offset and end are checked against config_len.
data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
.unwrap();
}
}
fn write_config(&mut self, offset: u64, data: &[u8]) {
let config_slice = self.config.as_mut_slice();
let data_len = data.len() as u64;
let config_len = config_slice.len() as u64;
if offset + data_len > config_len {
error!("Failed to write config space");
return;
}
let (_, right) = config_slice.split_at_mut(offset as usize);
right.copy_from_slice(&data[..]);
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
self.queue_sizes.len(),
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
// Save the interrupt EventFD as we need to return it on reset
// but clone it to pass into the thread.
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
let vu_call_evt_queue_list = setup_vhost_user(
&mut self.vu,
&mem.memory(),
queues,
queue_evts,
&interrupt_cb,
self.acked_features,
)
.map_err(ActivateError::VhostUserSetup)?;
// Initialize slave communication.
let slave_req_handler = if self.slave_req_support {
if let Some(cache) = self.cache.as_ref() {
let vu_master_req_handler = Arc::new(Mutex::new(SlaveReqHandler {
cache_offset: cache.0.addr,
cache_size: cache.0.len,
mmap_cache_addr: cache.0.host_addr,
mem,
}));
let req_handler = MasterReqHandler::new(vu_master_req_handler).map_err(|e| {
ActivateError::VhostUserSetup(Error::MasterReqHandlerCreation(e))
})?;
self.vu
.set_slave_request_fd(req_handler.get_tx_raw_fd())
.map_err(|e| {
ActivateError::VhostUserSetup(Error::VhostUserSetSlaveRequestFd(e))
})?;
Some(req_handler)
} else {
None
}
} else {
None
};
let mut handler = VhostUserEpollHandler::new(VhostUserEpollConfig {
vu_interrupt_list: vu_call_evt_queue_list,
interrupt_cb,
kill_evt,
pause_evt,
slave_req_handler,
});
let paused = self.paused.clone();
let mut epoll_threads = Vec::new();
thread::Builder::new()
.name("virtio_fs".to_string())
.spawn(move || handler.run(paused))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?;
self.epoll_threads = Some(epoll_threads);
Ok(())
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
if let Err(e) = reset_vhost_user(&mut self.vu, self.queue_sizes.len()) {
error!("Failed to reset vhost-user daemon: {:?}", e);
return None;
}
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
fn shutdown(&mut self) {
let _ = unsafe { libc::close(self.vu.as_raw_fd()) };
}
fn get_shm_regions(&self) -> Option<VirtioSharedMemoryList> {
if let Some(cache) = self.cache.as_ref() {
Some(cache.0.clone())
} else {
None
}
}
fn set_shm_regions(
&mut self,
shm_regions: VirtioSharedMemoryList,
) -> std::result::Result<(), crate::Error> {
if let Some(mut cache) = self.cache.as_mut() {
cache.0 = shm_regions;
Ok(())
} else {
Err(crate::Error::SetShmRegionsNotSupported)
}
}
fn update_memory(&mut self, mem: &GuestMemoryMmap) -> std::result::Result<(), crate::Error> {
update_mem_table(&mut self.vu, mem).map_err(crate::Error::VhostUserUpdateMemory)
}
fn userspace_mappings(&self) -> Vec<UserspaceMapping> {
let mut mappings = Vec::new();
if let Some(cache) = self.cache.as_ref() {
mappings.push(UserspaceMapping {
host_addr: cache.0.host_addr,
mem_slot: cache.0.mem_slot,
addr: cache.0.addr,
len: cache.0.len,
mergeable: false,
})
}
mappings
}
}
virtio_pausable!(Fs);
impl Snapshottable for Fs {
fn id(&self) -> String {
self.id.clone()
}
}
impl Transportable for Fs {}
impl Migratable for Fs {}

196
vm-virtio/src/vhost_user/handler.rs
View File

@@ -1,196 +0,0 @@
// Copyright (c) 2019 Intel Corporation. All rights reserved.
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-BSD-3-Clause file.
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
use super::super::{Queue, VirtioInterruptType};
use super::{Error, Result};
use vmm_sys_util::eventfd::EventFd;
use crate::VirtioInterrupt;
use std::fs::File;
use std::io;
use std::os::unix::io::{AsRawFd, FromRawFd};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use vhost_rs::vhost_user::{MasterReqHandler, VhostUserMasterReqHandler};
/// Collection of common parameters required by vhost-user devices while
/// call Epoll handler.
///
/// # Arguments
/// * `interrupt_cb` interrupt for virtqueue change.
/// * `kill_evt` - EventFd used to kill the vhost-user device.
/// * `vu_interrupt_list` - virtqueue and EventFd to signal when buffer used.
pub struct VhostUserEpollConfig<S: VhostUserMasterReqHandler> {
pub interrupt_cb: Arc<dyn VirtioInterrupt>,
pub kill_evt: EventFd,
pub pause_evt: EventFd,
pub vu_interrupt_list: Vec<(Option<EventFd>, Queue)>,
pub slave_req_handler: Option<MasterReqHandler<S>>,
}
pub struct VhostUserEpollHandler<S: VhostUserMasterReqHandler> {
vu_epoll_cfg: VhostUserEpollConfig<S>,
}
impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
/// Construct a new event handler for vhost-user based devices.
///
/// # Arguments
/// * `vu_epoll_cfg` - collection of common parameters for vhost-user devices
///
/// # Return
/// * `VhostUserEpollHandler` - epoll handler for vhost-user based devices
pub fn new(vu_epoll_cfg: VhostUserEpollConfig<S>) -> VhostUserEpollHandler<S> {
VhostUserEpollHandler { vu_epoll_cfg }
}
fn signal_used_queue(&self, queue: &Queue) -> Result<()> {
self.vu_epoll_cfg
.interrupt_cb
.trigger(&VirtioInterruptType::Queue, Some(queue))
.map_err(Error::FailedSignalingUsedQueue)
}
pub fn run(&mut self, paused: Arc<AtomicBool>) -> Result<()> {
// Create the epoll file descriptor
let epoll_fd = epoll::create(true).map_err(Error::EpollCreateFd)?;
// Use 'File' to enforce closing on 'epoll_fd'
let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
for (index, vhost_user_interrupt) in self.vu_epoll_cfg.vu_interrupt_list.iter().enumerate()
{
if let Some(eventfd) = &vhost_user_interrupt.0 {
// Add events
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
eventfd.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
)
.map_err(Error::EpollCtl)?;
}
}
let kill_evt_index = self.vu_epoll_cfg.vu_interrupt_list.len();
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.vu_epoll_cfg.kill_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, kill_evt_index as u64),
)
.map_err(Error::EpollCtl)?;
let pause_evt_index = kill_evt_index + 1;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self.vu_epoll_cfg.pause_evt.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, pause_evt_index as u64),
)
.map_err(Error::EpollCtl)?;
let mut index = pause_evt_index;
let slave_evt_index = if let Some(self_req_handler) = &self.vu_epoll_cfg.slave_req_handler {
index = pause_evt_index + 1;
epoll::ctl(
epoll_file.as_raw_fd(),
epoll::ControlOptions::EPOLL_CTL_ADD,
self_req_handler.as_raw_fd(),
epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
)
.map_err(Error::EpollCtl)?;
Some(index)
} else {
None
};
let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); index + 1];
// Before jumping into the epoll loop, check if the device is expected
// to be in a paused state. This is helpful for the restore code path
// as the device thread should not start processing anything before the
// device has been resumed.
while paused.load(Ordering::SeqCst) {
thread::park();
}
'poll: loop {
let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
Ok(res) => res,
Err(e) => {
if e.kind() == io::ErrorKind::Interrupted {
// It's well defined from the epoll_wait() syscall
// documentation that the epoll loop can be interrupted
// before any of the requested events occurred or the
// timeout expired. In both those cases, epoll_wait()
// returns an error of type EINTR, but this should not
// be considered as a regular error. Instead it is more
// appropriate to retry, by calling into epoll_wait().
continue;
}
return Err(Error::EpollWait(e));
}
};
for event in events.iter().take(num_events) {
let ev_type = event.data as usize;
match ev_type {
x if x < kill_evt_index => {
if let Some(eventfd) = &self.vu_epoll_cfg.vu_interrupt_list[x].0 {
eventfd.read().map_err(Error::FailedReadingQueue)?;
if let Err(e) =
self.signal_used_queue(&self.vu_epoll_cfg.vu_interrupt_list[x].1)
{
error!("Failed to signal used queue: {:?}", e);
break 'poll;
}
}
}
x if kill_evt_index == x => {
debug!("KILL_EVENT received, stopping epoll loop");
break 'poll;
}
x if pause_evt_index == x => {
debug!("PAUSE_EVENT received, pausing vhost-user epoll loop");
// We loop here to handle spurious park() returns.
// Until we have not resumed, the paused boolean will
// be true.
while paused.load(Ordering::SeqCst) {
thread::park();
}
// Drain pause event after the device has been resumed.
// This ensures the pause event has been seen by each
// and every thread related to this virtio device.
let _ = self.vu_epoll_cfg.pause_evt.read();
}
x if (slave_evt_index.is_some() && slave_evt_index.unwrap() == x) => {
if let Some(slave_req_handler) =
self.vu_epoll_cfg.slave_req_handler.as_mut()
{
slave_req_handler
.handle_request()
.map_err(Error::VhostUserSlaveRequest)?;
}
}
_ => {
error!("Unknown event for vhost-user");
}
}
}
}
Ok(())
}
}

102
vm-virtio/src/vhost_user/mod.rs
View File

@@ -1,102 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
extern crate epoll;
extern crate net_util;
extern crate vhost_rs;
extern crate virtio_bindings;
extern crate vm_memory;
use std::io;
use vhost_rs::Error as VhostError;
use vm_memory::Error as MmapError;
pub mod blk;
pub mod fs;
mod handler;
pub mod net;
pub mod vu_common_ctrl;
pub use self::blk::Blk;
pub use self::fs::*;
pub use self::net::Net;
pub use self::vu_common_ctrl::VhostUserConfig;
#[derive(Debug)]
pub enum Error {
/// Invalid available address.
AvailAddress,
/// Queue number is not correct
BadQueueNum,
/// Creating kill eventfd failed.
CreateKillEventFd(io::Error),
/// Cloning kill eventfd failed.
CloneKillEventFd(io::Error),
/// Invalid descriptor table address.
DescriptorTableAddress,
/// Create Epoll eventfd failed
EpollCreateFd(io::Error),
/// Epoll ctl error
EpollCtl(io::Error),
/// Epoll wait error
EpollWait(io::Error),
/// Read queue failed.
FailedReadingQueue(io::Error),
/// Signal used queue failed.
FailedSignalingUsedQueue(io::Error),
/// Failed to read vhost eventfd.
MemoryRegions(MmapError),
/// Failed to create master.
VhostUserCreateMaster(VhostError),
/// Failed to open vhost device.
VhostUserOpen(VhostError),
/// Connection to socket failed.
VhostUserConnect(vhost_rs::Error),
/// Get features failed.
VhostUserGetFeatures(VhostError),
/// Get queue max number failed.
VhostUserGetQueueMaxNum(VhostError),
/// Get protocol features failed.
VhostUserGetProtocolFeatures(VhostError),
/// Vhost-user Backend not support vhost-user protocol.
VhostUserProtocolNotSupport,
/// Set owner failed.
VhostUserSetOwner(VhostError),
/// Reset owner failed.
VhostUserResetOwner(VhostError),
/// Set features failed.
VhostUserSetFeatures(VhostError),
/// Set protocol features failed.
VhostUserSetProtocolFeatures(VhostError),
/// Set mem table failed.
VhostUserSetMemTable(VhostError),
/// Set vring num failed.
VhostUserSetVringNum(VhostError),
/// Set vring addr failed.
VhostUserSetVringAddr(VhostError),
/// Set vring base failed.
VhostUserSetVringBase(VhostError),
/// Set vring call failed.
VhostUserSetVringCall(VhostError),
/// Set vring kick failed.
VhostUserSetVringKick(VhostError),
/// Set vring enable failed.
VhostUserSetVringEnable(VhostError),
/// Failed to create vhost eventfd.
VhostIrqCreate(io::Error),
/// Failed to read vhost eventfd.
VhostIrqRead(io::Error),
/// Failed to read vhost eventfd.
VhostUserMemoryRegion(MmapError),
/// Failed to handle vhost-user slave request.
VhostUserSlaveRequest(vhost_rs::vhost_user::Error),
/// Failed to create the master request handler from slave.
MasterReqHandlerCreation(vhost_rs::vhost_user::Error),
/// Set slave request fd failed.
VhostUserSetSlaveRequestFd(vhost_rs::Error),
/// Invalid used address.
UsedAddress,
/// Invalid features provided from vhost-user backend
InvalidFeatures,
}
type Result<T> = std::result::Result<T, Error>;

375
vm-virtio/src/vhost_user/net.rs
View File

@@ -1,375 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
use super::super::net_util::{
build_net_config_space, CtrlVirtio, NetCtrlEpollHandler, VirtioNetConfig,
};
use super::super::Error as CtrlError;
use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
use super::handler::*;
use super::vu_common_ctrl::*;
use super::Error as DeviceError;
use super::{Error, Result};
use crate::VirtioInterrupt;
use libc::EFD_NONBLOCK;
use net_util::MacAddr;
use std::cmp;
use std::io::Write;
use std::os::unix::io::AsRawFd;
use std::result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use std::vec::Vec;
use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
use vhost_rs::VhostBackend;
use virtio_bindings::bindings::virtio_net;
use virtio_bindings::bindings::virtio_ring;
use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
use vmm_sys_util::eventfd::EventFd;
const DEFAULT_QUEUE_NUMBER: usize = 2;
struct SlaveReqHandler {}
impl VhostUserMasterReqHandler for SlaveReqHandler {}
pub struct Net {
id: String,
vhost_user_net: Master,
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
avail_features: u64,
acked_features: u64,
backend_features: u64,
config: VirtioNetConfig,
queue_sizes: Vec<u16>,
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
ctrl_queue_epoll_thread: Option<thread::JoinHandle<result::Result<(), CtrlError>>>,
paused: Arc<AtomicBool>,
}
impl Net {
/// Create a new vhost-user-net device
/// Create a new vhost-user-net device
pub fn new(id: String, mac_addr: MacAddr, vu_cfg: VhostUserConfig) -> Result<Net> {
let mut vhost_user_net = Master::connect(&vu_cfg.socket, vu_cfg.num_queues as u64)
.map_err(Error::VhostUserCreateMaster)?;
// Filling device and vring features VMM supports.
let mut avail_features = 1 << virtio_net::VIRTIO_NET_F_GUEST_CSUM
| 1 << virtio_net::VIRTIO_NET_F_CSUM
| 1 << virtio_net::VIRTIO_NET_F_GUEST_TSO4
| 1 << virtio_net::VIRTIO_NET_F_GUEST_TSO6
| 1 << virtio_net::VIRTIO_NET_F_GUEST_ECN
| 1 << virtio_net::VIRTIO_NET_F_GUEST_UFO
| 1 << virtio_net::VIRTIO_NET_F_HOST_TSO4
| 1 << virtio_net::VIRTIO_NET_F_HOST_TSO6
| 1 << virtio_net::VIRTIO_NET_F_HOST_ECN
| 1 << virtio_net::VIRTIO_NET_F_HOST_UFO
| 1 << virtio_net::VIRTIO_NET_F_MRG_RXBUF
| 1 << virtio_net::VIRTIO_F_NOTIFY_ON_EMPTY
| 1 << virtio_net::VIRTIO_F_VERSION_1
| 1 << virtio_ring::VIRTIO_RING_F_EVENT_IDX
| VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
vhost_user_net
.set_owner()
.map_err(Error::VhostUserSetOwner)?;
// Get features from backend, do negotiation to get a feature collection which
// both VMM and backend support.
let backend_features = vhost_user_net
.get_features()
.map_err(Error::VhostUserGetFeatures)?;
avail_features &= backend_features;
// Set features back is required by the vhost crate mechanism, since the
// later vhost call will check if features is filled in master before execution.
vhost_user_net
.set_features(avail_features)
.map_err(Error::VhostUserSetFeatures)?;
let protocol_features;
let mut acked_features = 0;
if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
protocol_features = vhost_user_net
.get_protocol_features()
.map_err(Error::VhostUserGetProtocolFeatures)?;
} else {
return Err(Error::VhostUserProtocolNotSupport);
}
let max_queue_number =
if protocol_features.bits() & VhostUserProtocolFeatures::MQ.bits() != 0 {
vhost_user_net
.set_protocol_features(protocol_features & VhostUserProtocolFeatures::MQ)
.map_err(Error::VhostUserSetProtocolFeatures)?;
match vhost_user_net.get_queue_num() {
Ok(qn) => qn,
Err(_) => DEFAULT_QUEUE_NUMBER as u64,
}
} else {
DEFAULT_QUEUE_NUMBER as u64
};
if vu_cfg.num_queues > max_queue_number as usize {
error!("vhost-user-net has queue number: {} larger than the max queue number: {} backend allowed\n",
vu_cfg.num_queues, max_queue_number);
return Err(Error::BadQueueNum);
}
avail_features |= 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ;
let queue_num = vu_cfg.num_queues + 1;
let mut config = VirtioNetConfig::default();
build_net_config_space(
&mut config,
mac_addr,
vu_cfg.num_queues,
&mut avail_features,
);
// Send set_vring_base here, since it could tell backends, like OVS + DPDK,
// how many virt queues to be handled, which backend required to know at early stage.
for i in 0..vu_cfg.num_queues {
vhost_user_net
.set_vring_base(i, 0)
.map_err(Error::VhostUserSetVringBase)?;
}
Ok(Net {
id,
vhost_user_net,
kill_evt: None,
pause_evt: None,
avail_features,
acked_features,
backend_features,
config,
queue_sizes: vec![vu_cfg.queue_size; queue_num],
queue_evts: None,
interrupt_cb: None,
epoll_threads: None,
ctrl_queue_epoll_thread: None,
paused: Arc::new(AtomicBool::new(false)),
})
}
}
impl Drop for Net {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
if let Err(e) = kill_evt.write(1) {
error!("failed to kill vhost-user-net: {:?}", e);
}
}
}
}
impl VirtioDevice for Net {
fn device_type(&self) -> u32 {
VirtioDeviceType::TYPE_NET as u32
}
fn queue_max_sizes(&self) -> &[u16] {
&self.queue_sizes
}
fn features(&self) -> u64 {
self.avail_features
}
fn ack_features(&mut self, value: u64) {
let mut v = value;
// Check if the guest is ACK'ing a feature that we didn't claim to have.
let unrequested_features = v & !self.avail_features;
if unrequested_features != 0 {
warn!("Received acknowledge request for unknown feature: {:x}", v);
// Don't count these features as acked.
v &= !unrequested_features;
}
self.acked_features |= v;
}
fn read_config(&self, offset: u64, mut data: &mut [u8]) {
let config_slice = self.config.as_slice();
let config_len = config_slice.len() as u64;
if offset >= config_len {
error!("Failed to read config space");
return;
}
if let Some(end) = offset.checked_add(data.len() as u64) {
// This write can't fail, offset and end are checked against config_len.
data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
.unwrap();
}
}
fn write_config(&mut self, offset: u64, data: &[u8]) {
let config_slice = self.config.as_mut_slice();
let data_len = data.len() as u64;
let config_len = config_slice.len() as u64;
if offset + data_len > config_len {
error!("Failed to write config space");
return;
}
let (_, right) = config_slice.split_at_mut(offset as usize);
right.copy_from_slice(&data[..]);
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
mut queues: Vec<Queue>,
mut queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
self.queue_sizes.len(),
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating kill EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.kill_evt = Some(self_kill_evt);
let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
error!("failed creating pause EventFd pair: {}", e);
ActivateError::BadActivate
})?;
self.pause_evt = Some(self_pause_evt);
// Save the interrupt EventFD as we need to return it on reset
// but clone it to pass into the thread.
self.interrupt_cb = Some(interrupt_cb.clone());
let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
for queue_evt in queue_evts.iter() {
// Save the queue EventFD as we need to return it on reset
// but clone it to pass into the thread.
tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?);
}
self.queue_evts = Some(tmp_queue_evts);
let queue_num = queue_evts.len();
if (self.acked_features & 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0
{
let cvq_queue = queues.remove(queue_num - 1);
let cvq_queue_evt = queue_evts.remove(queue_num - 1);
let mut ctrl_handler = NetCtrlEpollHandler {
mem: mem.clone(),
kill_evt: kill_evt.try_clone().unwrap(),
pause_evt: pause_evt.try_clone().unwrap(),
ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
epoll_fd: 0,
};
let paused = self.paused.clone();
thread::Builder::new()
.name("virtio_net".to_string())
.spawn(move || ctrl_handler.run_ctrl(paused))
.map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
.map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?;
}
let mut vu_interrupt_list = setup_vhost_user(
&mut self.vhost_user_net,
&mem.memory(),
queues,
queue_evts,
&interrupt_cb,
self.acked_features & self.backend_features,
)
.map_err(ActivateError::VhostUserNetSetup)?;
let mut epoll_threads = Vec::new();
for _ in 0..vu_interrupt_list.len() / 2 {
let mut interrupt_list_sub: Vec<(Option<EventFd>, Queue)> = Vec::with_capacity(2);
interrupt_list_sub.push(vu_interrupt_list.remove(0));
interrupt_list_sub.push(vu_interrupt_list.remove(0));
let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
interrupt_cb: interrupt_cb.clone(),
kill_evt: kill_evt.try_clone().unwrap(),
pause_evt: pause_evt.try_clone().unwrap(),
vu_interrupt_list: interrupt_list_sub,
slave_req_handler: None,
});
let paused = self.paused.clone();
thread::Builder::new()
.name("vhost_user_net".to_string())
.spawn(move || handler.run(paused))
.map(|thread| epoll_threads.push(thread))
.map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?;
}
self.epoll_threads = Some(epoll_threads);
Ok(())
}
fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
// We first must resume the virtio thread if it was paused.
if self.pause_evt.take().is_some() {
self.resume().ok()?;
}
if let Err(e) = reset_vhost_user(&mut self.vhost_user_net, self.queue_sizes.len()) {
error!("Failed to reset vhost-user daemon: {:?}", e);
return None;
}
if let Some(kill_evt) = self.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
// Return the interrupt and queue EventFDs
Some((
self.interrupt_cb.take().unwrap(),
self.queue_evts.take().unwrap(),
))
}
fn shutdown(&mut self) {
let _ = unsafe { libc::close(self.vhost_user_net.as_raw_fd()) };
}
fn update_memory(&mut self, mem: &GuestMemoryMmap) -> std::result::Result<(), crate::Error> {
update_mem_table(&mut self.vhost_user_net, mem).map_err(crate::Error::VhostUserUpdateMemory)
}
}
virtio_ctrl_q_pausable!(Net);
impl Snapshottable for Net {
fn id(&self) -> String {
self.id.clone()
}
}
impl Transportable for Net {}
impl Migratable for Net {}

148
vm-virtio/src/vhost_user/vu_common_ctrl.rs
View File

@@ -1,148 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
use super::super::Queue;
use super::{Error, Result};
use crate::queue::Descriptor;
use crate::{VirtioInterrupt, VirtioInterruptType};
use libc::EFD_NONBLOCK;
use std::convert::TryInto;
use std::os::unix::io::AsRawFd;
use std::sync::Arc;
use std::vec::Vec;
use vfio_ioctls::get_host_address_range;
use vhost_rs::vhost_user::{Master, VhostUserMaster};
use vhost_rs::{VhostBackend, VhostUserMemoryRegionInfo, VringConfigData};
use vm_memory::{Address, Error as MmapError, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
use vmm_sys_util::eventfd::EventFd;
#[derive(Debug, Clone)]
pub struct VhostUserConfig {
pub socket: String,
pub num_queues: usize,
pub queue_size: u16,
}
pub fn update_mem_table(vu: &mut Master, mem: &GuestMemoryMmap) -> Result<()> {
let mut regions: Vec<VhostUserMemoryRegionInfo> = Vec::new();
mem.with_regions_mut(|_, region| {
let (mmap_handle, mmap_offset) = match region.file_offset() {
Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()),
None => return Err(MmapError::NoMemoryRegion),
};
let vhost_user_net_reg = VhostUserMemoryRegionInfo {
guest_phys_addr: region.start_addr().raw_value(),
memory_size: region.len() as u64,
userspace_addr: region.as_ptr() as u64,
mmap_offset,
mmap_handle,
};
regions.push(vhost_user_net_reg);
Ok(())
})
.map_err(Error::VhostUserMemoryRegion)?;
vu.set_mem_table(regions.as_slice())
.map_err(Error::VhostUserSetMemTable)?;
Ok(())
}
pub fn setup_vhost_user_vring(
vu: &mut Master,
mem: &GuestMemoryMmap,
queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
virtio_interrupt: &Arc<dyn VirtioInterrupt>,
) -> Result<Vec<(Option<EventFd>, Queue)>> {
// Let's first provide the memory table to the backend.
update_mem_table(vu, mem)?;
let mut vu_interrupt_list = Vec::new();
for (queue_index, queue) in queues.into_iter().enumerate() {
let actual_size: usize = queue.actual_size().try_into().unwrap();
vu.set_vring_num(queue_index, queue.actual_size())
.map_err(Error::VhostUserSetVringNum)?;
let config_data = VringConfigData {
queue_max_size: queue.get_max_size(),
queue_size: queue.actual_size(),
flags: 0u32,
desc_table_addr: get_host_address_range(
mem,
queue.desc_table,
actual_size * std::mem::size_of::<Descriptor>(),
)
.ok_or_else(|| Error::DescriptorTableAddress)? as u64,
// The used ring is {flags: u16; idx: u16; virtq_used_elem [{id: u16, len: u16}; actual_size]},
// i.e. 4 + (4 + 4) * actual_size.
used_ring_addr: get_host_address_range(mem, queue.used_ring, 4 + actual_size * 8)
.ok_or_else(|| Error::UsedAddress)? as u64,
// The used ring is {flags: u16; idx: u16; elem [u16; actual_size]},
// i.e. 4 + (2) * actual_size.
avail_ring_addr: get_host_address_range(mem, queue.avail_ring, 4 + actual_size * 2)
.ok_or_else(|| Error::AvailAddress)? as u64,
log_addr: None,
};
vu.set_vring_addr(queue_index, &config_data)
.map_err(Error::VhostUserSetVringAddr)?;
vu.set_vring_base(queue_index, 0u16)
.map_err(Error::VhostUserSetVringBase)?;
if let Some(eventfd) = virtio_interrupt.notifier(&VirtioInterruptType::Queue, Some(&queue))
{
vu.set_vring_call(queue_index, &eventfd)
.map_err(Error::VhostUserSetVringCall)?;
vu_interrupt_list.push((None, queue));
} else {
let eventfd = EventFd::new(EFD_NONBLOCK).map_err(Error::VhostIrqCreate)?;
vu.set_vring_call(queue_index, &eventfd)
.map_err(Error::VhostUserSetVringCall)?;
vu_interrupt_list.push((Some(eventfd), queue));
}
vu.set_vring_kick(queue_index, &queue_evts[queue_index])
.map_err(Error::VhostUserSetVringKick)?;
vu.set_vring_enable(queue_index, true)
.map_err(Error::VhostUserSetVringEnable)?;
}
Ok(vu_interrupt_list)
}
pub fn setup_vhost_user(
vu: &mut Master,
mem: &GuestMemoryMmap,
queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
virtio_interrupt: &Arc<dyn VirtioInterrupt>,
acked_features: u64,
) -> Result<Vec<(Option<EventFd>, Queue)>> {
// Set features based on the acked features from the guest driver.
vu.set_features(acked_features)
.map_err(Error::VhostUserSetFeatures)?;
setup_vhost_user_vring(vu, mem, queues, queue_evts, virtio_interrupt)
}
pub fn reset_vhost_user(vu: &mut Master, num_queues: usize) -> Result<()> {
for queue_index in 0..num_queues {
// Disable the vrings.
vu.set_vring_enable(queue_index, false)
.map_err(Error::VhostUserSetVringEnable)?;
// Stop the vrings.
vu.get_vring_base(queue_index)
.map_err(Error::VhostUserSetFeatures)?;
}
// Reset the owner.
vu.reset_owner().map_err(Error::VhostUserResetOwner)
}

1249
vm-virtio/src/vsock/csm/connection.rs
View File

File diff suppressed because it is too large Load Diff

129
vm-virtio/src/vsock/csm/mod.rs
View File

@@ -1,129 +0,0 @@
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
/// This module implements our vsock connection state machine. The heavy lifting is done by
/// `connection::VsockConnection`, while this file only defines some constants and helper structs.
///
mod connection;
mod txbuf;
pub use connection::VsockConnection;
pub mod defs {
/// Vsock connection TX buffer capacity.
pub const CONN_TX_BUF_SIZE: u32 = 64 * 1024;
/// When the guest thinks we have less than this amount of free buffer space,
/// we will send them a credit update packet.
pub const CONN_CREDIT_UPDATE_THRESHOLD: u32 = 4 * 1024;
/// Connection request timeout, in millis.
pub const CONN_REQUEST_TIMEOUT_MS: u64 = 2000;
/// Connection graceful shutdown timeout, in millis.
pub const CONN_SHUTDOWN_TIMEOUT_MS: u64 = 2000;
}
#[derive(Debug)]
pub enum Error {
/// Attempted to push data to a full TX buffer.
TxBufFull,
/// An I/O error occurred, when attempting to flush the connection TX buffer.
TxBufFlush(std::io::Error),
/// An I/O error occurred, when attempting to write data to the host-side stream.
StreamWrite(std::io::Error),
}
type Result<T> = std::result::Result<T, Error>;
/// A vsock connection state.
///
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum ConnState {
/// The connection has been initiated by the host end, but is yet to be confirmed by the guest.
LocalInit,
/// The connection has been initiated by the guest, but we are yet to confirm it, by sending
/// a response packet (VSOCK_OP_RESPONSE).
PeerInit,
/// The connection handshake has been performed successfully, and data can now be exchanged.
Established,
/// The host (AF_UNIX) socket was closed.
LocalClosed,
/// A VSOCK_OP_SHUTDOWN packet was received from the guest. The tuple represents the guest R/W
/// indication: (will_not_recv_anymore_data, will_not_send_anymore_data).
PeerClosed(bool, bool),
/// The connection is scheduled to be forcefully terminated as soon as possible.
Killed,
}
/// An RX indication, used by `VsockConnection` to schedule future `recv_pkt()` responses.
/// For instance, after being notified that there is available data to be read from the host stream
/// (via `notify()`), the connection will store a `PendingRx::Rw` to be later inspected by
/// `recv_pkt()`.
///
#[derive(Clone, Copy, PartialEq)]
enum PendingRx {
/// We need to yield a connection request packet (VSOCK_OP_REQUEST).
Request = 0,
/// We need to yield a connection response packet (VSOCK_OP_RESPONSE).
Response = 1,
/// We need to yield a forceful connection termination packet (VSOCK_OP_RST).
Rst = 2,
/// We need to yield a data packet (VSOCK_OP_RW), by reading from the AF_UNIX socket.
Rw = 3,
/// We need to yield a credit update packet (VSOCK_OP_CREDIT_UPDATE).
CreditUpdate = 4,
}
impl PendingRx {
/// Transform the enum value into a bitmask, that can be used for set operations.
///
fn into_mask(self) -> u16 {
1u16 << (self as u16)
}
}
/// A set of RX indications (`PendingRx` items).
///
struct PendingRxSet {
data: u16,
}
impl PendingRxSet {
/// Insert an item into the set.
///
fn insert(&mut self, it: PendingRx) {
self.data |= it.into_mask();
}
/// Remove an item from the set and return:
/// - true, if the item was in the set; or
/// - false, if the item wasn't in the set.
///
fn remove(&mut self, it: PendingRx) -> bool {
let ret = self.contains(it);
self.data &= !it.into_mask();
ret
}
/// Check if an item is present in this set.
///
fn contains(&self, it: PendingRx) -> bool {
self.data & it.into_mask() != 0
}
/// Check if the set is empty.
///
fn is_empty(&self) -> bool {
self.data == 0
}
}
/// Create a set containing only one item.
///
impl From<PendingRx> for PendingRxSet {
fn from(it: PendingRx) -> Self {
Self {
data: it.into_mask(),
}
}
}

283
vm-virtio/src/vsock/csm/txbuf.rs
View File

@@ -1,283 +0,0 @@
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
use std::io::Write;
use std::mem;
use std::num::Wrapping;
use super::defs;
use super::{Error, Result};
/// A simple ring-buffer implementation, used by vsock connections to buffer TX (guest -> host)
/// data. Memory for this buffer is allocated lazily, since buffering will only be needed when
/// the host can't read fast enough.
///
pub struct TxBuf {
/// The actual u8 buffer - only allocated after the first push.
data: Option<Box<[u8; Self::SIZE]>>,
/// Ring-buffer head offset - where new data is pushed to.
head: Wrapping<u32>,
/// Ring-buffer tail offset - where data is flushed from.
tail: Wrapping<u32>,
}
impl TxBuf {
/// Total buffer size, in bytes.
///
const SIZE: usize = defs::CONN_TX_BUF_SIZE as usize;
/// Ring-buffer constructor.
///
pub fn new() -> Self {
Self {
data: None,
head: Wrapping(0),
tail: Wrapping(0),
}
}
/// Get the used length of this buffer - number of bytes that have been pushed in, but not
/// yet flushed out.
///
pub fn len(&self) -> usize {
(self.head - self.tail).0 as usize
}
/// Push a byte slice onto the ring-buffer.
///
/// Either the entire source slice will be pushed to the ring-buffer, or none of it, if
/// there isn't enough room, in which case `Err(Error::TxBufFull)` is returned.
///
pub fn push(&mut self, src: &[u8]) -> Result<()> {
// Error out if there's no room to push the entire slice.
if self.len() + src.len() > Self::SIZE {
return Err(Error::TxBufFull);
}
// We're using a closure here to return the boxed slice, instead of a value (i.e.
// `get_or_insert_with()` instead of `get_or_insert()`), because we only want the box
// created when `self.data` is None. If we were to use `get_or_insert(box)`, the box
// argument would always get evaluated (which implies a heap allocation), even though
// it would later be discarded (when `self.data.is_some()`). Apparently, clippy fails
// to see this, and insists on issuing some warning.
#[allow(clippy::redundant_closure)]
let data = self.data.get_or_insert_with(||
// Using uninitialized memory here is quite safe, since we never read from any
// area of the buffer before writing to it. First we push, then we flush only
// what had been prviously pushed.
Box::new(unsafe {mem::MaybeUninit::<[u8; Self::SIZE]>::uninit().assume_init()}));
// Buffer head, as an offset into the data slice.
let head_ofs = self.head.0 as usize % Self::SIZE;
// Pushing a slice to this buffer can take either one or two slice copies: - one copy,
// if the slice fits between `head_ofs` and `Self::SIZE`; or - two copies, if the
// ring-buffer head wraps around.
// First copy length: we can only go from the head offset up to the total buffer size.
let len = std::cmp::min(Self::SIZE - head_ofs, src.len());
data[head_ofs..(head_ofs + len)].copy_from_slice(&src[..len]);
// If the slice didn't fit, the buffer head will wrap around, and pushing continues
// from the start of the buffer (`&self.data[0]`).
if len < src.len() {
data[..(src.len() - len)].copy_from_slice(&src[len..]);
}
// Either way, we've just pushed exactly `src.len()` bytes, so that's the amount by
// which the (wrapping) buffer head needs to move forward.
self.head += Wrapping(src.len() as u32);
Ok(())
}
/// Flush the contents of the ring-buffer to a writable stream.
///
/// Return the number of bytes that have been transferred out of the ring-buffer and into
/// the writable stream.
///
pub fn flush_to<W>(&mut self, sink: &mut W) -> Result<usize>
where
W: Write,
{
// Nothing to do, if this buffer holds no data.
if self.is_empty() {
return Ok(0);
}
// Buffer tail, as an offset into the buffer data slice.
let tail_ofs = self.tail.0 as usize % Self::SIZE;
// Flushing the buffer can take either one or two writes:
// - one write, if the tail doesn't need to wrap around to reach the head; or
// - two writes, if the tail would wrap around: tail to slice end, then slice end to
// head.
// First write length: the lesser of tail to slice end, or tail to head.
let len_to_write = std::cmp::min(Self::SIZE - tail_ofs, self.len());
// It's safe to unwrap here, since we've already checked if the buffer was empty.
let data = self.data.as_ref().unwrap();
// Issue the first write and absorb any `WouldBlock` error (we can just try again
// later).
let written = sink
.write(&data[tail_ofs..(tail_ofs + len_to_write)])
.map_err(Error::TxBufFlush)?;
// Move the buffer tail ahead by the amount (of bytes) we were able to flush out.
self.tail += Wrapping(written as u32);
// If we weren't able to flush out as much as we tried, there's no point in attempting
// our second write.
if written < len_to_write {
return Ok(written);
}
// Attempt our second write. This will return immediately if a second write isn't
// needed, since checking for an empty buffer is the first thing we do in this
// function.
//
// Interesting corner case: if we've already written some data in the first pass,
// and then the second write fails, we will consider the flush action a success
// and return the number of bytes written in the first pass.
Ok(written + self.flush_to(sink).unwrap_or(0))
}
/// Check if the buffer holds any data that hasn't yet been flushed out.
///
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Error as IoError;
use std::io::Result as IoResult;
use std::io::{ErrorKind, Write};
struct TestSink {
data: Vec<u8>,
err: Option<IoError>,
capacity: usize,
}
impl TestSink {
const DEFAULT_CAPACITY: usize = 2 * TxBuf::SIZE;
fn new() -> Self {
Self {
data: Vec::with_capacity(Self::DEFAULT_CAPACITY),
err: None,
capacity: Self::DEFAULT_CAPACITY,
}
}
}
impl Write for TestSink {
fn write(&mut self, src: &[u8]) -> IoResult<usize> {
if self.err.is_some() {
return Err(self.err.take().unwrap());
}
let len_to_push = std::cmp::min(self.capacity - self.data.len(), src.len());
self.data.extend_from_slice(&src[..len_to_push]);
Ok(len_to_push)
}
fn flush(&mut self) -> IoResult<()> {
Ok(())
}
}
impl TestSink {
fn clear(&mut self) {
self.data = Vec::with_capacity(self.capacity);
self.err = None;
}
fn set_err(&mut self, err: IoError) {
self.err = Some(err);
}
fn set_capacity(&mut self, capacity: usize) {
self.capacity = capacity;
if self.data.len() > self.capacity {
self.data.resize(self.capacity, 0);
}
}
}
#[test]
fn test_push_nowrap() {
let mut txbuf = TxBuf::new();
let mut sink = TestSink::new();
assert!(txbuf.is_empty());
assert!(txbuf.data.is_none());
txbuf.push(&[1, 2, 3, 4]).unwrap();
txbuf.push(&[5, 6, 7, 8]).unwrap();
txbuf.flush_to(&mut sink).unwrap();
assert_eq!(sink.data, [1, 2, 3, 4, 5, 6, 7, 8]);
}
#[test]
fn test_push_wrap() {
let mut txbuf = TxBuf::new();
let mut sink = TestSink::new();
let mut tmp: Vec<u8> = Vec::new();
tmp.resize(TxBuf::SIZE - 2, 0);
txbuf.push(tmp.as_slice()).unwrap();
txbuf.flush_to(&mut sink).unwrap();
sink.clear();
txbuf.push(&[1, 2, 3, 4]).unwrap();
assert_eq!(txbuf.flush_to(&mut sink).unwrap(), 4);
assert_eq!(sink.data, [1, 2, 3, 4]);
}
#[test]
fn test_push_error() {
let mut txbuf = TxBuf::new();
let mut tmp = Vec::with_capacity(TxBuf::SIZE);
tmp.resize(TxBuf::SIZE - 1, 0);
txbuf.push(tmp.as_slice()).unwrap();
match txbuf.push(&[1, 2]) {
Err(Error::TxBufFull) => (),
other => panic!("Unexpected result: {:?}", other),
}
}
#[test]
fn test_incomplete_flush() {
let mut txbuf = TxBuf::new();
let mut sink = TestSink::new();
sink.set_capacity(2);
txbuf.push(&[1, 2, 3, 4]).unwrap();
assert_eq!(txbuf.flush_to(&mut sink).unwrap(), 2);
assert_eq!(txbuf.len(), 2);
assert_eq!(sink.data, [1, 2]);
sink.set_capacity(4);
assert_eq!(txbuf.flush_to(&mut sink).unwrap(), 2);
assert!(txbuf.is_empty());
assert_eq!(sink.data, [1, 2, 3, 4]);
}
#[test]
fn test_flush_error() {
const EACCESS: i32 = 13;
let mut txbuf = TxBuf::new();
let mut sink = TestSink::new();
txbuf.push(&[1, 2, 3, 4]).unwrap();
let io_err = IoError::from_raw_os_error(EACCESS);
sink.set_err(io_err);
match txbuf.flush_to(&mut sink) {
Err(Error::TxBufFlush(ref err)) if err.kind() == ErrorKind::PermissionDenied => (),
other => panic!("Unexpected result: {:?}", other),
}
}
}

1018
vm-virtio/src/vsock/device.rs
View File

File diff suppressed because it is too large Load Diff

361
vm-virtio/src/vsock/mod.rs
View File

@@ -1,361 +0,0 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Portions Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the THIRD-PARTY file.
mod csm;
mod device;
mod packet;
mod unix;
pub use self::device::Vsock;
pub use self::unix::VsockUnixBackend;
pub use self::unix::VsockUnixError;
pub use packet::VsockPacket;
use std::os::unix::io::RawFd;
mod defs {
/// Max vsock packet data/buffer size.
pub const MAX_PKT_BUF_SIZE: usize = 64 * 1024;
pub mod uapi {
/// Vsock packet operation IDs.
/// Defined in `/include/uapi/linux/virtio_vsock.h`.
///
/// Connection request.
pub const VSOCK_OP_REQUEST: u16 = 1;
/// Connection response.
pub const VSOCK_OP_RESPONSE: u16 = 2;
/// Connection reset.
pub const VSOCK_OP_RST: u16 = 3;
/// Connection clean shutdown.
pub const VSOCK_OP_SHUTDOWN: u16 = 4;
/// Connection data (read/write).
pub const VSOCK_OP_RW: u16 = 5;
/// Flow control credit update.
pub const VSOCK_OP_CREDIT_UPDATE: u16 = 6;
/// Flow control credit update request.
pub const VSOCK_OP_CREDIT_REQUEST: u16 = 7;
/// Vsock packet flags.
/// Defined in `/include/uapi/linux/virtio_vsock.h`.
///
/// Valid with a VSOCK_OP_SHUTDOWN packet: the packet sender will receive no more data.
pub const VSOCK_FLAGS_SHUTDOWN_RCV: u32 = 1;
/// Valid with a VSOCK_OP_SHUTDOWN packet: the packet sender will send no more data.
pub const VSOCK_FLAGS_SHUTDOWN_SEND: u32 = 2;
/// Vsock packet type.
/// Defined in `/include/uapi/linux/virtio_vsock.h`.
///
/// Stream / connection-oriented packet (the only currently valid type).
pub const VSOCK_TYPE_STREAM: u16 = 1;
pub const VSOCK_HOST_CID: u64 = 2;
}
}
#[derive(Debug)]
pub enum VsockError {
/// The vsock data/buffer virtio descriptor length is smaller than expected.
BufDescTooSmall,
/// The vsock data/buffer virtio descriptor is expected, but missing.
BufDescMissing,
/// Chained GuestMemory error.
GuestMemory,
/// Bounds check failed on guest memory pointer.
GuestMemoryBounds,
/// The vsock header descriptor length is too small.
HdrDescTooSmall(u32),
/// The vsock header `len` field holds an invalid value.
InvalidPktLen(u32),
/// A data fetch was attempted when no data was available.
NoData,
/// A data buffer was expected for the provided packet, but it is missing.
PktBufMissing,
/// Encountered an unexpected write-only virtio descriptor.
UnreadableDescriptor,
/// Encountered an unexpected read-only virtio descriptor.
UnwritableDescriptor,
}
type Result<T> = std::result::Result<T, VsockError>;
#[derive(Debug)]
pub enum VsockEpollHandlerError {
/// The vsock data/buffer virtio descriptor length is smaller than expected.
BufDescTooSmall,
/// The vsock data/buffer virtio descriptor is expected, but missing.
BufDescMissing,
/// Chained GuestMemory error.
GuestMemory,
/// Bounds check failed on guest memory pointer.
GuestMemoryBounds,
/// The vsock header descriptor length is too small.
HdrDescTooSmall(u32),
/// The vsock header `len` field holds an invalid value.
InvalidPktLen(u32),
/// A data fetch was attempted when no data was available.
NoData,
/// A data buffer was expected for the provided packet, but it is missing.
PktBufMissing,
/// Encountered an unexpected write-only virtio descriptor.
UnreadableDescriptor,
/// Encountered an unexpected read-only virtio descriptor.
UnwritableDescriptor,
}
/// A passive, event-driven object, that needs to be notified whenever an epoll-able event occurs.
/// An event-polling control loop will use `get_polled_fd()` and `get_polled_evset()` to query
/// the listener for the file descriptor and the set of events it's interested in. When such an
/// event occurs, the control loop will route the event to the listener via `notify()`.
///
pub trait VsockEpollListener {
/// Get the file descriptor the listener needs polled.
fn get_polled_fd(&self) -> RawFd;
/// Get the set of events for which the listener wants to be notified.
fn get_polled_evset(&self) -> epoll::Events;
/// Notify the listener that one ore more events have occurred.
fn notify(&mut self, evset: epoll::Events);
}
/// Any channel that handles vsock packet traffic: sending and receiving packets. Since we're
/// implementing the device model here, our responsibility is to always process the sending of
/// packets (i.e. the TX queue). So, any locally generated data, addressed to the driver (e.g.
/// a connection response or RST), will have to be queued, until we get to processing the RX queue.
///
/// Note: `recv_pkt()` and `send_pkt()` are named analogous to `Read::read()` and `Write::write()`,
/// respectively. I.e.
/// - `recv_pkt(&mut pkt)` will read data from the channel, and place it into `pkt`; and
/// - `send_pkt(&pkt)` will fetch data from `pkt`, and place it into the channel.
pub trait VsockChannel {
/// Read/receive an incoming packet from the channel.
fn recv_pkt(&mut self, pkt: &mut VsockPacket) -> Result<()>;
/// Write/send a packet through the channel.
fn send_pkt(&mut self, pkt: &VsockPacket) -> Result<()>;
/// Checks whether there is pending incoming data inside the channel, meaning that a subsequent
/// call to `recv_pkt()` won't fail.
fn has_pending_rx(&self) -> bool;
}
/// The vsock backend, which is basically an epoll-event-driven vsock channel, that needs to be
/// sendable through a mpsc channel (the latter due to how `vmm::EpollContext` works).
/// Currently, the only implementation we have is `crate::virtio::unix::muxer::VsockMuxer`, which
/// translates guest-side vsock connections to host-side Unix domain socket connections.
pub trait VsockBackend: VsockChannel + VsockEpollListener + Send {}
#[cfg(test)]
mod tests {
use super::device::{VsockEpollHandler, RX_QUEUE_EVENT, TX_QUEUE_EVENT};
use super::packet::VSOCK_PKT_HDR_SIZE;
use super::*;
use crate::device::{VirtioInterrupt, VirtioInterruptType};
use crate::queue::tests::VirtQueue as GuestQ;
use crate::queue::Queue;
use crate::{VIRTQ_DESC_F_NEXT, VIRTQ_DESC_F_WRITE};
use libc::EFD_NONBLOCK;
use std::os::unix::io::AsRawFd;
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, RwLock};
use vm_memory::{GuestAddress, GuestMemoryAtomic, GuestMemoryMmap};
use vmm_sys_util::eventfd::EventFd;
pub struct NoopVirtioInterrupt {}
impl VirtioInterrupt for NoopVirtioInterrupt {
fn trigger(
&self,
_int_type: &VirtioInterruptType,
_queue: Option<&Queue>,
) -> std::result::Result<(), std::io::Error> {
Ok(())
}
}
pub struct TestBackend {
pub evfd: EventFd,
pub rx_err: Option<VsockError>,
pub tx_err: Option<VsockError>,
pub pending_rx: bool,
pub rx_ok_cnt: usize,
pub tx_ok_cnt: usize,
pub evset: Option<epoll::Events>,
}
impl TestBackend {
pub fn new() -> Self {
Self {
evfd: EventFd::new(EFD_NONBLOCK).unwrap(),
rx_err: None,
tx_err: None,
pending_rx: false,
rx_ok_cnt: 0,
tx_ok_cnt: 0,
evset: None,
}
}
pub fn set_rx_err(&mut self, err: Option<VsockError>) {
self.rx_err = err;
}
pub fn set_tx_err(&mut self, err: Option<VsockError>) {
self.tx_err = err;
}
pub fn set_pending_rx(&mut self, prx: bool) {
self.pending_rx = prx;
}
}
impl VsockChannel for TestBackend {
fn recv_pkt(&mut self, _pkt: &mut VsockPacket) -> Result<()> {
match self.rx_err.take() {
None => {
self.rx_ok_cnt += 1;
Ok(())
}
Some(e) => Err(e),
}
}
fn send_pkt(&mut self, _pkt: &VsockPacket) -> Result<()> {
match self.tx_err.take() {
None => {
self.tx_ok_cnt += 1;
Ok(())
}
Some(e) => Err(e),
}
}
fn has_pending_rx(&self) -> bool {
self.pending_rx
}
}
impl VsockEpollListener for TestBackend {
fn get_polled_fd(&self) -> RawFd {
self.evfd.as_raw_fd()
}
fn get_polled_evset(&self) -> epoll::Events {
epoll::Events::EPOLLIN
}
fn notify(&mut self, evset: epoll::Events) {
self.evset = Some(evset);
}
}
impl VsockBackend for TestBackend {}
pub struct TestContext {
pub cid: u64,
pub mem: GuestMemoryMmap,
pub mem_size: usize,
pub device: Vsock<TestBackend>,
}
impl TestContext {
pub fn new() -> Self {
const CID: u64 = 52;
const MEM_SIZE: usize = 1024 * 1024 * 128;
Self {
cid: CID,
mem: GuestMemoryMmap::from_ranges(&[(GuestAddress(0), MEM_SIZE)]).unwrap(),
mem_size: MEM_SIZE,
device: Vsock::new(
String::from("vsock"),
CID,
PathBuf::from("/test/sock"),
TestBackend::new(),
false,
)
.unwrap(),
}
}
pub fn create_epoll_handler_context(&self) -> EpollHandlerContext {
const QSIZE: u16 = 2;
let guest_rxvq = GuestQ::new(GuestAddress(0x0010_0000), &self.mem, QSIZE as u16);
let guest_txvq = GuestQ::new(GuestAddress(0x0020_0000), &self.mem, QSIZE as u16);
let guest_evvq = GuestQ::new(GuestAddress(0x0030_0000), &self.mem, QSIZE as u16);
let rxvq = guest_rxvq.create_queue();
let txvq = guest_txvq.create_queue();
let evvq = guest_evvq.create_queue();
// Set up one available descriptor in the RX queue.
guest_rxvq.dtable[0].set(
0x0040_0000,
VSOCK_PKT_HDR_SIZE as u32,
VIRTQ_DESC_F_WRITE | VIRTQ_DESC_F_NEXT,
1,
);
guest_rxvq.dtable[1].set(0x0040_1000, 4096, VIRTQ_DESC_F_WRITE, 0);
guest_rxvq.avail.ring[0].set(0);
guest_rxvq.avail.idx.set(1);
// Set up one available descriptor in the TX queue.
guest_txvq.dtable[0].set(0x0050_0000, VSOCK_PKT_HDR_SIZE as u32, VIRTQ_DESC_F_NEXT, 1);
guest_txvq.dtable[1].set(0x0050_1000, 4096, 0, 0);
guest_txvq.avail.ring[0].set(0);
guest_txvq.avail.idx.set(1);
let queues = vec![rxvq, txvq, evvq];
let queue_evts = vec![
EventFd::new(EFD_NONBLOCK).unwrap(),
EventFd::new(EFD_NONBLOCK).unwrap(),
EventFd::new(EFD_NONBLOCK).unwrap(),
];
let interrupt_cb = Arc::new(NoopVirtioInterrupt {});
EpollHandlerContext {
guest_rxvq,
guest_txvq,
guest_evvq,
handler: VsockEpollHandler {
mem: GuestMemoryAtomic::new(self.mem.clone()),
queues,
queue_evts,
kill_evt: EventFd::new(EFD_NONBLOCK).unwrap(),
pause_evt: EventFd::new(EFD_NONBLOCK).unwrap(),
interrupt_cb,
backend: Arc::new(RwLock::new(TestBackend::new())),
},
}
}
}
pub struct EpollHandlerContext<'a> {
pub handler: VsockEpollHandler<TestBackend>,
pub guest_rxvq: GuestQ<'a>,
pub guest_txvq: GuestQ<'a>,
pub guest_evvq: GuestQ<'a>,
}
impl<'a> EpollHandlerContext<'a> {
pub fn signal_txq_event(&mut self) {
self.handler.queue_evts[1].write(1).unwrap();
self.handler
.handle_event(
TX_QUEUE_EVENT,
epoll::Events::EPOLLIN,
Arc::new(AtomicBool::new(false)),
)
.unwrap();
}
pub fn signal_rxq_event(&mut self) {
self.handler.queue_evts[0].write(1).unwrap();
self.handler
.handle_event(
RX_QUEUE_EVENT,
epoll::Events::EPOLLIN,
Arc::new(AtomicBool::new(false)),
)
.unwrap();
}
}
}

656
vm-virtio/src/vsock/packet.rs
View File

@@ -1,656 +0,0 @@
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
/// `VsockPacket` provides a thin wrapper over the buffers exchanged via virtio queues.
/// There are two components to a vsock packet, each using its own descriptor in a
/// virtio queue:
/// - the packet header; and
/// - the packet data/buffer.
/// There is a 1:1 relation between descriptor chains and packets: the first (chain head) holds
/// the header, and an optional second descriptor holds the data. The second descriptor is only
/// present for data packets (VSOCK_OP_RW).
///
/// `VsockPacket` wraps these two buffers and provides direct access to the data stored
/// in guest memory. This is done to avoid unnecessarily copying data from guest memory
/// to temporary buffers, before passing it on to the vsock backend.
///
use byteorder::{ByteOrder, LittleEndian};
use super::super::DescriptorChain;
use super::defs;
use super::{Result, VsockError};
use vfio_ioctls::get_host_address_range;
// The vsock packet header is defined by the C struct:
//
// ```C
// struct virtio_vsock_hdr {
// le64 src_cid;
// le64 dst_cid;
// le32 src_port;
// le32 dst_port;
// le32 len;
// le16 type;
// le16 op;
// le32 flags;
// le32 buf_alloc;
// le32 fwd_cnt;
// };
// ```
//
// This structed will occupy the buffer pointed to by the head descriptor. We'll be accessing it
// as a byte slice. To that end, we define below the offsets for each field struct, as well as the
// packed struct size, as a bunch of `usize` consts.
// Note that these offsets are only used privately by the `VsockPacket` struct, the public interface
// consisting of getter and setter methods, for each struct field, that will also handle the correct
// endianess.
/// The vsock packet header struct size (when packed).
pub const VSOCK_PKT_HDR_SIZE: usize = 44;
// Source CID.
const HDROFF_SRC_CID: usize = 0;
// Destination CID.
const HDROFF_DST_CID: usize = 8;
// Source port.
const HDROFF_SRC_PORT: usize = 16;
// Destination port.
const HDROFF_DST_PORT: usize = 20;
// Data length (in bytes) - may be 0, if there is no data buffer.
const HDROFF_LEN: usize = 24;
// Socket type. Currently, only connection-oriented streams are defined by the vsock protocol.
const HDROFF_TYPE: usize = 28;
// Operation ID - one of the VSOCK_OP_* values; e.g.
// - VSOCK_OP_RW: a data packet;
// - VSOCK_OP_REQUEST: connection request;
// - VSOCK_OP_RST: forcefull connection termination;
// etc (see `super::defs::uapi` for the full list).
const HDROFF_OP: usize = 30;
// Additional options (flags) associated with the current operation (`op`).
// Currently, only used with shutdown requests (VSOCK_OP_SHUTDOWN).
const HDROFF_FLAGS: usize = 32;
// Size (in bytes) of the packet sender receive buffer (for the connection to which this packet
// belongs).
const HDROFF_BUF_ALLOC: usize = 36;
// Number of bytes the sender has received and consumed (for the connection to which this packet
// belongs). For instance, for our Unix backend, this counter would be the total number of bytes
// we have successfully written to a backing Unix socket.
const HDROFF_FWD_CNT: usize = 40;
/// The vsock packet, implemented as a wrapper over a virtq descriptor chain:
/// - the chain head, holding the packet header; and
/// - (an optional) data/buffer descriptor, only present for data packets (VSOCK_OP_RW).
///
pub struct VsockPacket {
hdr: *mut u8,
buf: Option<*mut u8>,
buf_size: usize,
}
impl VsockPacket {
/// Create the packet wrapper from a TX virtq chain head.
///
/// The chain head is expected to hold valid packet header data. A following packet buffer
/// descriptor can optionally end the chain. Bounds and pointer checks are performed when
/// creating the wrapper.
///
pub fn from_tx_virtq_head(head: &DescriptorChain) -> Result<Self> {
// All buffers in the TX queue must be readable.
//
if head.is_write_only() {
return Err(VsockError::UnreadableDescriptor);
}
// The packet header should fit inside the head descriptor.
if head.len < VSOCK_PKT_HDR_SIZE as u32 {
return Err(VsockError::HdrDescTooSmall(head.len));
}
let mut pkt = Self {
hdr: get_host_address_range(head.mem, head.addr, VSOCK_PKT_HDR_SIZE)
.ok_or_else(|| VsockError::GuestMemory)? as *mut u8,
buf: None,
buf_size: 0,
};
// No point looking for a data/buffer descriptor, if the packet is zero-lengthed.
if pkt.is_empty() {
return Ok(pkt);
}
// Reject weirdly-sized packets.
//
if pkt.len() > defs::MAX_PKT_BUF_SIZE as u32 {
return Err(VsockError::InvalidPktLen(pkt.len()));
}
// If the packet header showed a non-zero length, there should be a data descriptor here.
let buf_desc = head.next_descriptor().ok_or(VsockError::BufDescMissing)?;
// TX data should be read-only.
if buf_desc.is_write_only() {
return Err(VsockError::UnreadableDescriptor);
}
// The data buffer should be large enough to fit the size of the data, as described by
// the header descriptor.
if buf_desc.len < pkt.len() {
return Err(VsockError::BufDescTooSmall);
}
pkt.buf_size = buf_desc.len as usize;
pkt.buf = Some(
get_host_address_range(buf_desc.mem, buf_desc.addr, pkt.buf_size)
.ok_or_else(|| VsockError::GuestMemory)? as *mut u8,
);
Ok(pkt)
}
/// Create the packet wrapper from an RX virtq chain head.
///
/// There must be two descriptors in the chain, both writable: a header descriptor and a data
/// descriptor. Bounds and pointer checks are performed when creating the wrapper.
///
pub fn from_rx_virtq_head(head: &DescriptorChain) -> Result<Self> {
// All RX buffers must be writable.
//
if !head.is_write_only() {
return Err(VsockError::UnwritableDescriptor);
}
// The packet header should fit inside the head descriptor.
if head.len < VSOCK_PKT_HDR_SIZE as u32 {
return Err(VsockError::HdrDescTooSmall(head.len));
}
// All RX descriptor chains should have a header and a data descriptor.
if !head.has_next() {
return Err(VsockError::BufDescMissing);
}
let buf_desc = head.next_descriptor().ok_or(VsockError::BufDescMissing)?;
let buf_size = buf_desc.len as usize;
Ok(Self {
hdr: get_host_address_range(head.mem, head.addr, VSOCK_PKT_HDR_SIZE)
.ok_or_else(|| VsockError::GuestMemory)? as *mut u8,
buf: Some(
get_host_address_range(buf_desc.mem, buf_desc.addr, buf_size)
.ok_or_else(|| VsockError::GuestMemory)? as *mut u8,
),
buf_size,
})
}
/// Provides in-place, byte-slice, access to the vsock packet header.
///
pub fn hdr(&self) -> &[u8] {
// This is safe since bound checks have already been performed when creating the packet
// from the virtq descriptor.
unsafe { std::slice::from_raw_parts(self.hdr as *const u8, VSOCK_PKT_HDR_SIZE) }
}
/// Provides in-place, byte-slice, mutable access to the vsock packet header.
///
pub fn hdr_mut(&mut self) -> &mut [u8] {
// This is safe since bound checks have already been performed when creating the packet
// from the virtq descriptor.
unsafe { std::slice::from_raw_parts_mut(self.hdr, VSOCK_PKT_HDR_SIZE) }
}
/// Provides in-place, byte-slice access to the vsock packet data buffer.
///
/// Note: control packets (e.g. connection request or reset) have no data buffer associated.
/// For those packets, this method will return `None`.
/// Also note: calling `len()` on the returned slice will yield the buffer size, which may be
/// (and often is) larger than the length of the packet data. The packet data length
/// is stored in the packet header, and accessible via `VsockPacket::len()`.
pub fn buf(&self) -> Option<&[u8]> {
self.buf.map(|ptr| {
// This is safe since bound checks have already been performed when creating the packet
// from the virtq descriptor.
unsafe { std::slice::from_raw_parts(ptr as *const u8, self.buf_size) }
})
}
/// Provides in-place, byte-slice, mutable access to the vsock packet data buffer.
///
/// Note: control packets (e.g. connection request or reset) have no data buffer associated.
/// For those packets, this method will return `None`.
/// Also note: calling `len()` on the returned slice will yield the buffer size, which may be
/// (and often is) larger than the length of the packet data. The packet data length
/// is stored in the packet header, and accessible via `VsockPacket::len()`.
pub fn buf_mut(&mut self) -> Option<&mut [u8]> {
self.buf.map(|ptr| {
// This is safe since bound checks have already been performed when creating the packet
// from the virtq descriptor.
unsafe { std::slice::from_raw_parts_mut(ptr, self.buf_size) }
})
}
pub fn src_cid(&self) -> u64 {
LittleEndian::read_u64(&self.hdr()[HDROFF_SRC_CID..])
}
pub fn set_src_cid(&mut self, cid: u64) -> &mut Self {
LittleEndian::write_u64(&mut self.hdr_mut()[HDROFF_SRC_CID..], cid);
self
}
pub fn dst_cid(&self) -> u64 {
LittleEndian::read_u64(&self.hdr()[HDROFF_DST_CID..])
}
pub fn set_dst_cid(&mut self, cid: u64) -> &mut Self {
LittleEndian::write_u64(&mut self.hdr_mut()[HDROFF_DST_CID..], cid);
self
}
pub fn src_port(&self) -> u32 {
LittleEndian::read_u32(&self.hdr()[HDROFF_SRC_PORT..])
}
pub fn set_src_port(&mut self, port: u32) -> &mut Self {
LittleEndian::write_u32(&mut self.hdr_mut()[HDROFF_SRC_PORT..], port);
self
}
pub fn dst_port(&self) -> u32 {
LittleEndian::read_u32(&self.hdr()[HDROFF_DST_PORT..])
}
pub fn set_dst_port(&mut self, port: u32) -> &mut Self {
LittleEndian::write_u32(&mut self.hdr_mut()[HDROFF_DST_PORT..], port);
self
}
pub fn len(&self) -> u32 {
LittleEndian::read_u32(&self.hdr()[HDROFF_LEN..])
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn set_len(&mut self, len: u32) -> &mut Self {
LittleEndian::write_u32(&mut self.hdr_mut()[HDROFF_LEN..], len);
self
}
pub fn type_(&self) -> u16 {
LittleEndian::read_u16(&self.hdr()[HDROFF_TYPE..])
}
pub fn set_type(&mut self, type_: u16) -> &mut Self {
LittleEndian::write_u16(&mut self.hdr_mut()[HDROFF_TYPE..], type_);
self
}
pub fn op(&self) -> u16 {
LittleEndian::read_u16(&self.hdr()[HDROFF_OP..])
}
pub fn set_op(&mut self, op: u16) -> &mut Self {
LittleEndian::write_u16(&mut self.hdr_mut()[HDROFF_OP..], op);
self
}
pub fn flags(&self) -> u32 {
LittleEndian::read_u32(&self.hdr()[HDROFF_FLAGS..])
}
pub fn set_flags(&mut self, flags: u32) -> &mut Self {
LittleEndian::write_u32(&mut self.hdr_mut()[HDROFF_FLAGS..], flags);
self
}
pub fn set_flag(&mut self, flag: u32) -> &mut Self {
self.set_flags(self.flags() | flag);
self
}
pub fn buf_alloc(&self) -> u32 {
LittleEndian::read_u32(&self.hdr()[HDROFF_BUF_ALLOC..])
}
pub fn set_buf_alloc(&mut self, buf_alloc: u32) -> &mut Self {
LittleEndian::write_u32(&mut self.hdr_mut()[HDROFF_BUF_ALLOC..], buf_alloc);
self
}
pub fn fwd_cnt(&self) -> u32 {
LittleEndian::read_u32(&self.hdr()[HDROFF_FWD_CNT..])
}
pub fn set_fwd_cnt(&mut self, fwd_cnt: u32) -> &mut Self {
LittleEndian::write_u32(&mut self.hdr_mut()[HDROFF_FWD_CNT..], fwd_cnt);
self
}
}
#[cfg(test)]
mod tests {
use vm_memory::{GuestAddress, GuestMemoryMmap};
use super::super::tests::TestContext;
use super::*;
use crate::queue::tests::VirtqDesc as GuestQDesc;
use crate::vsock::defs::MAX_PKT_BUF_SIZE;
use crate::VIRTQ_DESC_F_WRITE;
macro_rules! create_context {
($test_ctx:ident, $handler_ctx:ident) => {
let $test_ctx = TestContext::new();
let mut $handler_ctx = $test_ctx.create_epoll_handler_context();
// For TX packets, hdr.len should be set to a valid value.
set_pkt_len(1024, &$handler_ctx.guest_txvq.dtable[0], &$test_ctx.mem);
};
}
macro_rules! expect_asm_error {
(tx, $test_ctx:expr, $handler_ctx:expr, $err:pat) => {
expect_asm_error!($test_ctx, $handler_ctx, $err, from_tx_virtq_head, 1);
};
(rx, $test_ctx:expr, $handler_ctx:expr, $err:pat) => {
expect_asm_error!($test_ctx, $handler_ctx, $err, from_rx_virtq_head, 0);
};
($test_ctx:expr, $handler_ctx:expr, $err:pat, $ctor:ident, $vq:expr) => {
match VsockPacket::$ctor(
&$handler_ctx.handler.queues[$vq]
.iter(&$test_ctx.mem)
.next()
.unwrap(),
) {
Err($err) => (),
Ok(_) => panic!("Packet assembly should've failed!"),
Err(other) => panic!("Packet assembly failed with: {:?}", other),
}
};
}
fn set_pkt_len(len: u32, guest_desc: &GuestQDesc, mem: &GuestMemoryMmap) {
let hdr_gpa = guest_desc.addr.get();
let hdr_ptr = get_host_address_range(mem, GuestAddress(hdr_gpa), VSOCK_PKT_HDR_SIZE)
.unwrap() as *mut u8;
let len_ptr = unsafe { hdr_ptr.add(HDROFF_LEN) };
LittleEndian::write_u32(unsafe { std::slice::from_raw_parts_mut(len_ptr, 4) }, len);
}
#[test]
#[allow(clippy::cognitive_complexity)]
fn test_tx_packet_assembly() {
// Test case: successful TX packet assembly.
{
create_context!(test_ctx, handler_ctx);
let pkt = VsockPacket::from_tx_virtq_head(
&handler_ctx.handler.queues[1]
.iter(&test_ctx.mem)
.next()
.unwrap(),
)
.unwrap();
assert_eq!(pkt.hdr().len(), VSOCK_PKT_HDR_SIZE);
assert_eq!(
pkt.buf().unwrap().len(),
handler_ctx.guest_txvq.dtable[1].len.get() as usize
);
}
// Test case: error on write-only hdr descriptor.
{
create_context!(test_ctx, handler_ctx);
handler_ctx.guest_txvq.dtable[0]
.flags
.set(VIRTQ_DESC_F_WRITE);
expect_asm_error!(tx, test_ctx, handler_ctx, VsockError::UnreadableDescriptor);
}
// Test case: header descriptor has insufficient space to hold the packet header.
{
create_context!(test_ctx, handler_ctx);
handler_ctx.guest_txvq.dtable[0]
.len
.set(VSOCK_PKT_HDR_SIZE as u32 - 1);
expect_asm_error!(tx, test_ctx, handler_ctx, VsockError::HdrDescTooSmall(_));
}
// Test case: zero-length TX packet.
{
create_context!(test_ctx, handler_ctx);
set_pkt_len(0, &handler_ctx.guest_txvq.dtable[0], &test_ctx.mem);
let mut pkt = VsockPacket::from_tx_virtq_head(
&handler_ctx.handler.queues[1]
.iter(&test_ctx.mem)
.next()
.unwrap(),
)
.unwrap();
assert!(pkt.buf().is_none());
assert!(pkt.buf_mut().is_none());
}
// Test case: TX packet has more data than we can handle.
{
create_context!(test_ctx, handler_ctx);
set_pkt_len(
MAX_PKT_BUF_SIZE as u32 + 1,
&handler_ctx.guest_txvq.dtable[0],
&test_ctx.mem,
);
expect_asm_error!(tx, test_ctx, handler_ctx, VsockError::InvalidPktLen(_));
}
// Test case:
// - packet header advertises some data length; and
// - the data descriptor is missing.
{
create_context!(test_ctx, handler_ctx);
set_pkt_len(1024, &handler_ctx.guest_txvq.dtable[0], &test_ctx.mem);
handler_ctx.guest_txvq.dtable[0].flags.set(0);
expect_asm_error!(tx, test_ctx, handler_ctx, VsockError::BufDescMissing);
}
// Test case: error on write-only buf descriptor.
{
create_context!(test_ctx, handler_ctx);
handler_ctx.guest_txvq.dtable[1]
.flags
.set(VIRTQ_DESC_F_WRITE);
expect_asm_error!(tx, test_ctx, handler_ctx, VsockError::UnreadableDescriptor);
}
// Test case: the buffer descriptor cannot fit all the data advertised by the the
// packet header `len` field.
{
create_context!(test_ctx, handler_ctx);
set_pkt_len(8 * 1024, &handler_ctx.guest_txvq.dtable[0], &test_ctx.mem);
handler_ctx.guest_txvq.dtable[1].len.set(4 * 1024);
expect_asm_error!(tx, test_ctx, handler_ctx, VsockError::BufDescTooSmall);
}
}
#[test]
fn test_rx_packet_assembly() {
// Test case: successful RX packet assembly.
{
create_context!(test_ctx, handler_ctx);
let pkt = VsockPacket::from_rx_virtq_head(
&handler_ctx.handler.queues[0]
.iter(&test_ctx.mem)
.next()
.unwrap(),
)
.unwrap();
assert_eq!(pkt.hdr().len(), VSOCK_PKT_HDR_SIZE);
assert_eq!(
pkt.buf().unwrap().len(),
handler_ctx.guest_rxvq.dtable[1].len.get() as usize
);
}
// Test case: read-only RX packet header.
{
create_context!(test_ctx, handler_ctx);
handler_ctx.guest_rxvq.dtable[0].flags.set(0);
expect_asm_error!(rx, test_ctx, handler_ctx, VsockError::UnwritableDescriptor);
}
// Test case: RX descriptor head cannot fit the entire packet header.
{
create_context!(test_ctx, handler_ctx);
handler_ctx.guest_rxvq.dtable[0]
.len
.set(VSOCK_PKT_HDR_SIZE as u32 - 1);
expect_asm_error!(rx, test_ctx, handler_ctx, VsockError::HdrDescTooSmall(_));
}
// Test case: RX descriptor chain is missing the packet buffer descriptor.
{
create_context!(test_ctx, handler_ctx);
handler_ctx.guest_rxvq.dtable[0]
.flags
.set(VIRTQ_DESC_F_WRITE);
expect_asm_error!(rx, test_ctx, handler_ctx, VsockError::BufDescMissing);
}
}
#[test]
#[allow(clippy::cognitive_complexity)]
fn test_packet_hdr_accessors() {
const SRC_CID: u64 = 1;
const DST_CID: u64 = 2;
const SRC_PORT: u32 = 3;
const DST_PORT: u32 = 4;
const LEN: u32 = 5;
const TYPE: u16 = 6;
const OP: u16 = 7;
const FLAGS: u32 = 8;
const BUF_ALLOC: u32 = 9;
const FWD_CNT: u32 = 10;
create_context!(test_ctx, handler_ctx);
let mut pkt = VsockPacket::from_rx_virtq_head(
&handler_ctx.handler.queues[0]
.iter(&test_ctx.mem)
.next()
.unwrap(),
)
.unwrap();
// Test field accessors.
pkt.set_src_cid(SRC_CID)
.set_dst_cid(DST_CID)
.set_src_port(SRC_PORT)
.set_dst_port(DST_PORT)
.set_len(LEN)
.set_type(TYPE)
.set_op(OP)
.set_flags(FLAGS)
.set_buf_alloc(BUF_ALLOC)
.set_fwd_cnt(FWD_CNT);
assert_eq!(pkt.src_cid(), SRC_CID);
assert_eq!(pkt.dst_cid(), DST_CID);
assert_eq!(pkt.src_port(), SRC_PORT);
assert_eq!(pkt.dst_port(), DST_PORT);
assert_eq!(pkt.len(), LEN);
assert_eq!(pkt.type_(), TYPE);
assert_eq!(pkt.op(), OP);
assert_eq!(pkt.flags(), FLAGS);
assert_eq!(pkt.buf_alloc(), BUF_ALLOC);
assert_eq!(pkt.fwd_cnt(), FWD_CNT);
// Test individual flag setting.
let flags = pkt.flags() | 0b1000;
pkt.set_flag(0b1000);
assert_eq!(pkt.flags(), flags);
// Test packet header as-slice access.
//
assert_eq!(pkt.hdr().len(), VSOCK_PKT_HDR_SIZE);
assert_eq!(
SRC_CID,
LittleEndian::read_u64(&pkt.hdr()[HDROFF_SRC_CID..])
);
assert_eq!(
DST_CID,
LittleEndian::read_u64(&pkt.hdr()[HDROFF_DST_CID..])
);
assert_eq!(
SRC_PORT,
LittleEndian::read_u32(&pkt.hdr()[HDROFF_SRC_PORT..])
);
assert_eq!(
DST_PORT,
LittleEndian::read_u32(&pkt.hdr()[HDROFF_DST_PORT..])
);
assert_eq!(LEN, LittleEndian::read_u32(&pkt.hdr()[HDROFF_LEN..]));
assert_eq!(TYPE, LittleEndian::read_u16(&pkt.hdr()[HDROFF_TYPE..]));
assert_eq!(OP, LittleEndian::read_u16(&pkt.hdr()[HDROFF_OP..]));
assert_eq!(FLAGS, LittleEndian::read_u32(&pkt.hdr()[HDROFF_FLAGS..]));
assert_eq!(
BUF_ALLOC,
LittleEndian::read_u32(&pkt.hdr()[HDROFF_BUF_ALLOC..])
);
assert_eq!(
FWD_CNT,
LittleEndian::read_u32(&pkt.hdr()[HDROFF_FWD_CNT..])
);
assert_eq!(pkt.hdr_mut().len(), VSOCK_PKT_HDR_SIZE);
for b in pkt.hdr_mut() {
*b = 0;
}
assert_eq!(pkt.src_cid(), 0);
assert_eq!(pkt.dst_cid(), 0);
assert_eq!(pkt.src_port(), 0);
assert_eq!(pkt.dst_port(), 0);
assert_eq!(pkt.len(), 0);
assert_eq!(pkt.type_(), 0);
assert_eq!(pkt.op(), 0);
assert_eq!(pkt.flags(), 0);
assert_eq!(pkt.buf_alloc(), 0);
assert_eq!(pkt.fwd_cnt(), 0);
}
#[test]
fn test_packet_buf() {
create_context!(test_ctx, handler_ctx);
let mut pkt = VsockPacket::from_rx_virtq_head(
&handler_ctx.handler.queues[0]
.iter(&test_ctx.mem)
.next()
.unwrap(),
)
.unwrap();
assert_eq!(
pkt.buf().unwrap().len(),
handler_ctx.guest_rxvq.dtable[1].len.get() as usize
);
assert_eq!(
pkt.buf_mut().unwrap().len(),
handler_ctx.guest_rxvq.dtable[1].len.get() as usize
);
for i in 0..pkt.buf().unwrap().len() {
pkt.buf_mut().unwrap()[i] = (i % 0x100) as u8;
assert_eq!(pkt.buf().unwrap()[i], (i % 0x100) as u8);
}
}
}

56
vm-virtio/src/vsock/unix/mod.rs
View File

@@ -1,56 +0,0 @@
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
/// This module implements the Unix Domain Sockets backend for vsock - a mediator between
/// guest-side AF_VSOCK sockets and host-side AF_UNIX sockets. The heavy lifting is performed by
/// `muxer::VsockMuxer`, a connection multiplexer that uses `super::csm::VsockConnection` for
/// handling vsock connection states.
/// Check out `muxer.rs` for a more detailed explanation of the inner workings of this backend.
///
mod muxer;
mod muxer_killq;
mod muxer_rxq;
pub use muxer::VsockMuxer as VsockUnixBackend;
pub use Error as VsockUnixError;
mod defs {
/// Maximum number of established connections that we can handle.
pub const MAX_CONNECTIONS: usize = 1023;
/// Size of the muxer RX packet queue.
pub const MUXER_RXQ_SIZE: usize = 256;
/// Size of the muxer connection kill queue.
pub const MUXER_KILLQ_SIZE: usize = 128;
}
#[derive(Debug)]
pub enum Error {
/// Error converting from UTF-8
ConvertFromUTF8(std::str::Utf8Error),
/// Error registering a new epoll-listening FD.
EpollAdd(std::io::Error),
/// Error creating an epoll FD.
EpollFdCreate(std::io::Error),
/// The host made an invalid vsock port connection request.
InvalidPortRequest,
/// Error parsing integer.
ParseInteger(std::num::ParseIntError),
/// Error reading stream port.
ReadStreamPort(Box<Error>),
/// Error accepting a new connection from the host-side Unix socket.
UnixAccept(std::io::Error),
/// Error binding to the host-side Unix socket.
UnixBind(std::io::Error),
/// Error connecting to a host-side Unix socket.
UnixConnect(std::io::Error),
/// Error reading from host-side Unix socket.
UnixRead(std::io::Error),
/// Muxer connection limit reached.
TooManyConnections,
}
type Result<T> = std::result::Result<T, Error>;
type MuxerConnection = super::csm::VsockConnection<std::os::unix::net::UnixStream>;

1418
vm-virtio/src/vsock/unix/muxer.rs
View File

File diff suppressed because it is too large Load Diff

140
vm-virtio/src/vsock/unix/muxer_killq.rs
View File

@@ -1,140 +0,0 @@
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
/// `MuxerKillQ` implements a helper object that `VsockMuxer` can use for scheduling forced
/// connection termination. I.e. after one peer issues a clean shutdown request
/// (VSOCK_OP_SHUTDOWN), the concerned connection is queued for termination (VSOCK_OP_RST) in
/// the near future (herein implemented via an expiring timer).
///
/// Whenever the muxer needs to schedule a connection for termination, it pushes it (or rather
/// an identifier - the connection key) to this queue. A subsequent pop() operation will
/// succeed if and only if the first connection in the queue is ready to be terminated (i.e.
/// its kill timer expired).
///
/// Without using this queue, the muxer would have to walk its entire connection pool
/// (hashmap), whenever it needs to check for expired kill timers. With this queue, both
/// scheduling and termination are performed in constant time. However, since we don't want to
/// waste space on a kill queue that's as big as the connection hashmap itself, it is possible
/// that this queue may become full at times. We call this kill queue "synchronized" if we are
/// certain that all connections that are awaiting termination are present in the queue. This
/// means a simple constant-time pop() operation is enough to check whether any connections
/// need to be terminated. When the kill queue becomes full, though, pushing fails, so
/// connections that should be terminated are left out. The queue is not synchronized anymore.
/// When that happens, the muxer will first drain the queue, and then replace it with a new
/// queue, created by walking the connection pool, looking for connections that will be
/// expiring in the future.
///
use std::collections::{HashMap, VecDeque};
use std::time::Instant;
use super::defs;
use super::muxer::ConnMapKey;
use super::MuxerConnection;
/// A kill queue item, holding the connection key and the scheduled time for termination.
///
#[derive(Clone, Copy)]
struct MuxerKillQItem {
key: ConnMapKey,
kill_time: Instant,
}
/// The connection kill queue: a FIFO structure, storing the connections that are scheduled for
/// termination.
///
pub struct MuxerKillQ {
/// The kill queue contents.
q: VecDeque<MuxerKillQItem>,
/// The kill queue sync status:
/// - when true, all connections that are awaiting termination are guaranteed to be in this
/// queue;
/// - when false, some connections may have been left out.
///
synced: bool,
}
impl MuxerKillQ {
const SIZE: usize = defs::MUXER_KILLQ_SIZE;
/// Trivial kill queue constructor.
///
pub fn new() -> Self {
Self {
q: VecDeque::with_capacity(Self::SIZE),
synced: true,
}
}
/// Create a kill queue by walking the connection pool, looking for connections that are
/// set to expire at some point in the future.
/// Note: if more than `Self::SIZE` connections are found, the queue will be created in an
/// out-of-sync state, and will be discarded after it is emptied.
///
pub fn from_conn_map(conn_map: &HashMap<ConnMapKey, MuxerConnection>) -> Self {
let mut q_buf: Vec<MuxerKillQItem> = Vec::with_capacity(Self::SIZE);
let mut synced = true;
for (key, conn) in conn_map.iter() {
if !conn.will_expire() {
continue;
}
if q_buf.len() >= Self::SIZE {
synced = false;
break;
}
q_buf.push(MuxerKillQItem {
key: *key,
kill_time: conn.expiry().unwrap(),
});
}
q_buf.sort_unstable_by_key(|it| it.kill_time);
Self {
q: q_buf.into(),
synced,
}
}
/// Push a connection key to the queue, scheduling it for termination at
/// `CONN_SHUTDOWN_TIMEOUT_MS` from now (the push time).
///
pub fn push(&mut self, key: ConnMapKey, kill_time: Instant) {
if !self.is_synced() || self.is_full() {
self.synced = false;
return;
}
self.q.push_back(MuxerKillQItem { key, kill_time });
}
/// Attempt to pop an expired connection from the kill queue.
///
/// This will succeed and return a connection key, only if the connection at the front of
/// the queue has expired. Otherwise, `None` is returned.
///
pub fn pop(&mut self) -> Option<ConnMapKey> {
if let Some(item) = self.q.front() {
if Instant::now() > item.kill_time {
return Some(self.q.pop_front().unwrap().key);
}
}
None
}
/// Check if the kill queue is synchronized with the connection pool.
///
pub fn is_synced(&self) -> bool {
self.synced
}
/// Check if the kill queue is empty, obviously.
///
pub fn is_empty(&self) -> bool {
self.q.len() == 0
}
/// Check if the kill queue is full.
///
pub fn is_full(&self) -> bool {
self.q.len() == Self::SIZE
}
}

145
vm-virtio/src/vsock/unix/muxer_rxq.rs
View File

@@ -1,145 +0,0 @@
// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
//
/// `MuxerRxQ` implements a helper object that `VsockMuxer` can use for queuing RX (host -> guest)
/// packets (or rather instructions on how to build said packets).
///
/// Under ideal operation, every connection, that has pending RX data, will be present in the muxer
/// RX queue. However, since the RX queue is smaller than the connection pool, it may, under some
/// conditions, become full, meaning that it can no longer account for all the connections that can
/// yield RX data. When that happens, we say that it is no longer "synchronized" (i.e. with the
/// connection pool). A desynchronized RX queue still holds valid data, and the muxer will
/// continue to pop packets from it. However, when a desynchronized queue is drained, additional
/// data may still be available, so the muxer will have to perform a more costly walk of the entire
/// connection pool to find it. This walk is performed here, as part of building an RX queue from
/// the connection pool. When an out-of-sync is drained, the muxer will discard it, and attempt to
/// rebuild a synced one.
///
use std::collections::{HashMap, VecDeque};
use super::super::VsockChannel;
use super::defs;
use super::muxer::{ConnMapKey, MuxerRx};
use super::MuxerConnection;
/// The muxer RX queue.
///
pub struct MuxerRxQ {
/// The RX queue data.
q: VecDeque<MuxerRx>,
/// The RX queue sync status.
synced: bool,
}
impl MuxerRxQ {
const SIZE: usize = defs::MUXER_RXQ_SIZE;
/// Trivial RX queue constructor.
///
pub fn new() -> Self {
Self {
q: VecDeque::with_capacity(Self::SIZE),
synced: true,
}
}
/// Attempt to build an RX queue, that is synchronized to the connection pool.
/// Note: the resulting queue may still be desynchronized, if there are too many connections
/// that have pending RX data. In that case, the muxer will first drain this queue, and
/// then try again to build a synchronized one.
///
pub fn from_conn_map(conn_map: &HashMap<ConnMapKey, MuxerConnection>) -> Self {
let mut q = VecDeque::new();
let mut synced = true;
for (key, conn) in conn_map.iter() {
if !conn.has_pending_rx() {
continue;
}
if q.len() >= Self::SIZE {
synced = false;
break;
}
q.push_back(MuxerRx::ConnRx(*key));
}
Self { q, synced }
}
/// Push a new RX item to the queue.
///
/// A push will fail when:
/// - trying to push a connection key onto an out-of-sync, or full queue; or
/// - trying to push an RST onto a queue already full of RSTs.
/// RSTs take precedence over connections, because connections can always be queried for
/// pending RX data later. Aside from this queue, there is no other storage for RSTs, so
/// failing to push one means that we have to drop the packet.
///
/// Returns:
/// - `true` if the new item has been successfully queued; or
/// - `false` if there was no room left in the queue.
///
pub fn push(&mut self, rx: MuxerRx) -> bool {
// Pushing to a non-full, synchronized queue will always succeed.
if self.is_synced() && !self.is_full() {
self.q.push_back(rx);
return true;
}
match rx {
MuxerRx::RstPkt { .. } => {
// If we just failed to push an RST packet, we'll look through the queue, trying to
// find a connection key that we could evict. This way, the queue does lose sync,
// but we don't drop any packets.
for qi in self.q.iter_mut().rev() {
if let MuxerRx::ConnRx(_) = qi {
*qi = rx;
self.synced = false;
return true;
}
}
}
MuxerRx::ConnRx(_) => {
self.synced = false;
}
};
false
}
/// Peek into the front of the queue.
///
pub fn peek(&self) -> Option<MuxerRx> {
self.q.front().copied()
}
/// Pop an RX item from the front of the queue.
///
pub fn pop(&mut self) -> Option<MuxerRx> {
self.q.pop_front()
}
/// Check if the RX queue is synchronized with the connection pool.
///
pub fn is_synced(&self) -> bool {
self.synced
}
/// Get the total number of items in the queue.
///
pub fn len(&self) -> usize {
self.q.len()
}
/// Check if the queue is empty.
///
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Check if the queue is full.
///
pub fn is_full(&self) -> bool {
self.len() == Self::SIZE
}
}