mirror of
https://github.com/cloud-hypervisor/cloud-hypervisor.git
synced 2026-08-05 02:19:16 +00:00
virtio-devices, vmm: Always restore virtio devices in paused state
Following the new restore design, it is not appropriate to set every virtio device threads into a paused state after they've been started. This is why we remove the line of code pausing the devices only after they've been restored, and replace it with a small patch in every virtio device implementation. When a virtio device is created as part of a restored VM, the associated "paused" boolean is set to true. This ensures the corresponding thread will be directly parked when being started, avoiding the thread to be in a different state than the one it was on the source VM during the snapshot. Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
This commit is contained in:
Sebastien Boeuf
@@ -13,6 +13,7 @@ use block_util::VirtioBlockConfig;
|
||||
use seccompiler::SeccompAction;
|
||||
use std::mem;
|
||||
use std::result;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::{Arc, Barrier, Mutex};
|
||||
use std::thread;
|
||||
use std::vec::Vec;
|
||||
@@ -79,96 +80,102 @@ impl Blk {
|
||||
let mut vu =
|
||||
VhostUserHandle::connect_vhost_user(false, &vu_cfg.socket, num_queues as u64, false)?;
|
||||
|
||||
let (avail_features, acked_features, acked_protocol_features, vu_num_queues, config) =
|
||||
if let Some(state) = state {
|
||||
info!("Restoring vhost-user-block {}", id);
|
||||
let (
|
||||
avail_features,
|
||||
acked_features,
|
||||
acked_protocol_features,
|
||||
vu_num_queues,
|
||||
config,
|
||||
paused,
|
||||
) = if let Some(state) = state {
|
||||
info!("Restoring vhost-user-block {}", id);
|
||||
|
||||
vu.set_protocol_features_vhost_user(
|
||||
state.acked_features,
|
||||
state.acked_protocol_features,
|
||||
)?;
|
||||
vu.set_protocol_features_vhost_user(
|
||||
state.acked_features,
|
||||
state.acked_protocol_features,
|
||||
)?;
|
||||
|
||||
(
|
||||
state.avail_features,
|
||||
state.acked_features,
|
||||
state.acked_protocol_features,
|
||||
state.vu_num_queues,
|
||||
state.config,
|
||||
)
|
||||
} else {
|
||||
// Filling device and vring features VMM supports.
|
||||
let mut avail_features = 1 << VIRTIO_BLK_F_SIZE_MAX
|
||||
| 1 << VIRTIO_BLK_F_SEG_MAX
|
||||
| 1 << VIRTIO_BLK_F_GEOMETRY
|
||||
| 1 << VIRTIO_BLK_F_RO
|
||||
| 1 << VIRTIO_BLK_F_BLK_SIZE
|
||||
| 1 << VIRTIO_BLK_F_FLUSH
|
||||
| 1 << VIRTIO_BLK_F_TOPOLOGY
|
||||
| 1 << VIRTIO_BLK_F_CONFIG_WCE
|
||||
| 1 << VIRTIO_BLK_F_DISCARD
|
||||
| 1 << VIRTIO_BLK_F_WRITE_ZEROES
|
||||
| DEFAULT_VIRTIO_FEATURES;
|
||||
(
|
||||
state.avail_features,
|
||||
state.acked_features,
|
||||
state.acked_protocol_features,
|
||||
state.vu_num_queues,
|
||||
state.config,
|
||||
true,
|
||||
)
|
||||
} else {
|
||||
// Filling device and vring features VMM supports.
|
||||
let mut avail_features = 1 << VIRTIO_BLK_F_SIZE_MAX
|
||||
| 1 << VIRTIO_BLK_F_SEG_MAX
|
||||
| 1 << VIRTIO_BLK_F_GEOMETRY
|
||||
| 1 << VIRTIO_BLK_F_RO
|
||||
| 1 << VIRTIO_BLK_F_BLK_SIZE
|
||||
| 1 << VIRTIO_BLK_F_FLUSH
|
||||
| 1 << VIRTIO_BLK_F_TOPOLOGY
|
||||
| 1 << VIRTIO_BLK_F_CONFIG_WCE
|
||||
| 1 << VIRTIO_BLK_F_DISCARD
|
||||
| 1 << VIRTIO_BLK_F_WRITE_ZEROES
|
||||
| DEFAULT_VIRTIO_FEATURES;
|
||||
|
||||
if num_queues > 1 {
|
||||
avail_features |= 1 << VIRTIO_BLK_F_MQ;
|
||||
}
|
||||
if num_queues > 1 {
|
||||
avail_features |= 1 << VIRTIO_BLK_F_MQ;
|
||||
}
|
||||
|
||||
let avail_protocol_features = VhostUserProtocolFeatures::CONFIG
|
||||
| VhostUserProtocolFeatures::MQ
|
||||
| VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS
|
||||
| VhostUserProtocolFeatures::REPLY_ACK
|
||||
| VhostUserProtocolFeatures::INFLIGHT_SHMFD
|
||||
| VhostUserProtocolFeatures::LOG_SHMFD;
|
||||
let avail_protocol_features = VhostUserProtocolFeatures::CONFIG
|
||||
| VhostUserProtocolFeatures::MQ
|
||||
| VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS
|
||||
| VhostUserProtocolFeatures::REPLY_ACK
|
||||
| VhostUserProtocolFeatures::INFLIGHT_SHMFD
|
||||
| VhostUserProtocolFeatures::LOG_SHMFD;
|
||||
|
||||
let (acked_features, acked_protocol_features) =
|
||||
vu.negotiate_features_vhost_user(avail_features, avail_protocol_features)?;
|
||||
let (acked_features, acked_protocol_features) =
|
||||
vu.negotiate_features_vhost_user(avail_features, avail_protocol_features)?;
|
||||
|
||||
let backend_num_queues =
|
||||
if acked_protocol_features & VhostUserProtocolFeatures::MQ.bits() != 0 {
|
||||
vu.socket_handle()
|
||||
.get_queue_num()
|
||||
.map_err(Error::VhostUserGetQueueMaxNum)?
|
||||
as usize
|
||||
} else {
|
||||
DEFAULT_QUEUE_NUMBER
|
||||
};
|
||||
let backend_num_queues =
|
||||
if acked_protocol_features & VhostUserProtocolFeatures::MQ.bits() != 0 {
|
||||
vu.socket_handle()
|
||||
.get_queue_num()
|
||||
.map_err(Error::VhostUserGetQueueMaxNum)? as usize
|
||||
} else {
|
||||
DEFAULT_QUEUE_NUMBER
|
||||
};
|
||||
|
||||
if num_queues > backend_num_queues {
|
||||
error!("vhost-user-blk requested too many queues ({}) since the backend only supports {}\n",
|
||||
if num_queues > backend_num_queues {
|
||||
error!("vhost-user-blk requested too many queues ({}) since the backend only supports {}\n",
|
||||
num_queues, backend_num_queues);
|
||||
return Err(Error::BadQueueNum);
|
||||
}
|
||||
return Err(Error::BadQueueNum);
|
||||
}
|
||||
|
||||
let config_len = mem::size_of::<VirtioBlockConfig>();
|
||||
let config_space: Vec<u8> = vec![0u8; config_len];
|
||||
let (_, config_space) = vu
|
||||
.socket_handle()
|
||||
.get_config(
|
||||
VHOST_USER_CONFIG_OFFSET,
|
||||
config_len as u32,
|
||||
VhostUserConfigFlags::WRITABLE,
|
||||
config_space.as_slice(),
|
||||
)
|
||||
.map_err(Error::VhostUserGetConfig)?;
|
||||
let mut config = VirtioBlockConfig::default();
|
||||
if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice())
|
||||
{
|
||||
config = *backend_config;
|
||||
config.num_queues = num_queues as u16;
|
||||
}
|
||||
|
||||
(
|
||||
acked_features,
|
||||
// If part of the available features that have been acked,
|
||||
// the PROTOCOL_FEATURES bit must be already set through
|
||||
// the VIRTIO acked features as we know the guest would
|
||||
// never ack it, thus the feature would be lost.
|
||||
acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(),
|
||||
acked_protocol_features,
|
||||
num_queues,
|
||||
config,
|
||||
let config_len = mem::size_of::<VirtioBlockConfig>();
|
||||
let config_space: Vec<u8> = vec![0u8; config_len];
|
||||
let (_, config_space) = vu
|
||||
.socket_handle()
|
||||
.get_config(
|
||||
VHOST_USER_CONFIG_OFFSET,
|
||||
config_len as u32,
|
||||
VhostUserConfigFlags::WRITABLE,
|
||||
config_space.as_slice(),
|
||||
)
|
||||
};
|
||||
.map_err(Error::VhostUserGetConfig)?;
|
||||
let mut config = VirtioBlockConfig::default();
|
||||
if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice()) {
|
||||
config = *backend_config;
|
||||
config.num_queues = num_queues as u16;
|
||||
}
|
||||
|
||||
(
|
||||
acked_features,
|
||||
// If part of the available features that have been acked,
|
||||
// the PROTOCOL_FEATURES bit must be already set through
|
||||
// the VIRTIO acked features as we know the guest would
|
||||
// never ack it, thus the feature would be lost.
|
||||
acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(),
|
||||
acked_protocol_features,
|
||||
num_queues,
|
||||
config,
|
||||
false,
|
||||
)
|
||||
};
|
||||
|
||||
Ok(Blk {
|
||||
common: VirtioCommon {
|
||||
@@ -178,6 +185,7 @@ impl Blk {
|
||||
acked_features,
|
||||
paused_sync: Some(Arc::new(Barrier::new(2))),
|
||||
min_queues: DEFAULT_QUEUE_NUMBER as u16,
|
||||
paused: Arc::new(AtomicBool::new(paused)),
|
||||
..Default::default()
|
||||
},
|
||||
vu_common: VhostUserCommon {
|
||||
|
||||
@@ -16,6 +16,7 @@ use seccompiler::SeccompAction;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::result;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::{Arc, Barrier, Mutex};
|
||||
use std::thread;
|
||||
use versionize::{VersionMap, Versionize, VersionizeResult};
|
||||
@@ -333,6 +334,7 @@ impl Fs {
|
||||
vu_num_queues,
|
||||
config,
|
||||
slave_req_support,
|
||||
paused,
|
||||
) = if let Some(state) = state {
|
||||
info!("Restoring vhost-user-fs {}", id);
|
||||
|
||||
@@ -348,6 +350,7 @@ impl Fs {
|
||||
state.vu_num_queues,
|
||||
state.config,
|
||||
state.slave_req_support,
|
||||
true,
|
||||
)
|
||||
} else {
|
||||
// Filling device and vring features VMM supports.
|
||||
@@ -407,6 +410,7 @@ impl Fs {
|
||||
num_queues,
|
||||
config,
|
||||
slave_req_support,
|
||||
false,
|
||||
)
|
||||
};
|
||||
|
||||
@@ -418,6 +422,7 @@ impl Fs {
|
||||
queue_sizes: vec![queue_size; num_queues],
|
||||
paused_sync: Some(Arc::new(Barrier::new(2))),
|
||||
min_queues: 1,
|
||||
paused: Arc::new(AtomicBool::new(paused)),
|
||||
..Default::default()
|
||||
},
|
||||
vu_common: VhostUserCommon {
|
||||
|
||||
@@ -13,6 +13,7 @@ use crate::{GuestMemoryMmap, GuestRegionMmap};
|
||||
use net_util::{build_net_config_space, CtrlQueue, MacAddr, VirtioNetConfig};
|
||||
use seccompiler::SeccompAction;
|
||||
use std::result;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::{Arc, Barrier, Mutex};
|
||||
use std::thread;
|
||||
use std::vec::Vec;
|
||||
@@ -83,106 +84,113 @@ impl Net {
|
||||
let mut vu =
|
||||
VhostUserHandle::connect_vhost_user(server, &vu_cfg.socket, num_queues as u64, false)?;
|
||||
|
||||
let (avail_features, acked_features, acked_protocol_features, vu_num_queues, config) =
|
||||
if let Some(state) = state {
|
||||
info!("Restoring vhost-user-net {}", id);
|
||||
let (
|
||||
avail_features,
|
||||
acked_features,
|
||||
acked_protocol_features,
|
||||
vu_num_queues,
|
||||
config,
|
||||
paused,
|
||||
) = if let Some(state) = state {
|
||||
info!("Restoring vhost-user-net {}", id);
|
||||
|
||||
// The backend acknowledged features must not contain
|
||||
// VIRTIO_NET_F_MAC since we don't expect the backend
|
||||
// to handle it.
|
||||
let backend_acked_features = state.acked_features & !(1 << VIRTIO_NET_F_MAC);
|
||||
// The backend acknowledged features must not contain
|
||||
// VIRTIO_NET_F_MAC since we don't expect the backend
|
||||
// to handle it.
|
||||
let backend_acked_features = state.acked_features & !(1 << VIRTIO_NET_F_MAC);
|
||||
|
||||
vu.set_protocol_features_vhost_user(
|
||||
backend_acked_features,
|
||||
state.acked_protocol_features,
|
||||
)?;
|
||||
vu.set_protocol_features_vhost_user(
|
||||
backend_acked_features,
|
||||
state.acked_protocol_features,
|
||||
)?;
|
||||
|
||||
// If the control queue feature has been negotiated, let's
|
||||
// increase the number of queues.
|
||||
if state.acked_features & (1 << VIRTIO_NET_F_CTRL_VQ) != 0 {
|
||||
num_queues += 1;
|
||||
}
|
||||
// If the control queue feature has been negotiated, let's
|
||||
// increase the number of queues.
|
||||
if state.acked_features & (1 << VIRTIO_NET_F_CTRL_VQ) != 0 {
|
||||
num_queues += 1;
|
||||
}
|
||||
|
||||
(
|
||||
state.avail_features,
|
||||
state.acked_features,
|
||||
state.acked_protocol_features,
|
||||
state.vu_num_queues,
|
||||
state.config,
|
||||
)
|
||||
} else {
|
||||
// Filling device and vring features VMM supports.
|
||||
let mut avail_features = 1 << VIRTIO_NET_F_CSUM
|
||||
| 1 << VIRTIO_NET_F_GUEST_CSUM
|
||||
| 1 << VIRTIO_NET_F_GUEST_TSO4
|
||||
| 1 << VIRTIO_NET_F_GUEST_TSO6
|
||||
| 1 << VIRTIO_NET_F_GUEST_ECN
|
||||
| 1 << VIRTIO_NET_F_GUEST_UFO
|
||||
| 1 << VIRTIO_NET_F_HOST_TSO4
|
||||
| 1 << VIRTIO_NET_F_HOST_TSO6
|
||||
| 1 << VIRTIO_NET_F_HOST_ECN
|
||||
| 1 << VIRTIO_NET_F_HOST_UFO
|
||||
| 1 << VIRTIO_NET_F_MRG_RXBUF
|
||||
| 1 << VIRTIO_NET_F_CTRL_VQ
|
||||
| 1 << VIRTIO_F_RING_EVENT_IDX
|
||||
| 1 << VIRTIO_F_VERSION_1
|
||||
| VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
|
||||
(
|
||||
state.avail_features,
|
||||
state.acked_features,
|
||||
state.acked_protocol_features,
|
||||
state.vu_num_queues,
|
||||
state.config,
|
||||
true,
|
||||
)
|
||||
} else {
|
||||
// Filling device and vring features VMM supports.
|
||||
let mut avail_features = 1 << VIRTIO_NET_F_CSUM
|
||||
| 1 << VIRTIO_NET_F_GUEST_CSUM
|
||||
| 1 << VIRTIO_NET_F_GUEST_TSO4
|
||||
| 1 << VIRTIO_NET_F_GUEST_TSO6
|
||||
| 1 << VIRTIO_NET_F_GUEST_ECN
|
||||
| 1 << VIRTIO_NET_F_GUEST_UFO
|
||||
| 1 << VIRTIO_NET_F_HOST_TSO4
|
||||
| 1 << VIRTIO_NET_F_HOST_TSO6
|
||||
| 1 << VIRTIO_NET_F_HOST_ECN
|
||||
| 1 << VIRTIO_NET_F_HOST_UFO
|
||||
| 1 << VIRTIO_NET_F_MRG_RXBUF
|
||||
| 1 << VIRTIO_NET_F_CTRL_VQ
|
||||
| 1 << VIRTIO_F_RING_EVENT_IDX
|
||||
| 1 << VIRTIO_F_VERSION_1
|
||||
| VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
|
||||
|
||||
if mtu.is_some() {
|
||||
avail_features |= 1u64 << VIRTIO_NET_F_MTU;
|
||||
}
|
||||
if mtu.is_some() {
|
||||
avail_features |= 1u64 << VIRTIO_NET_F_MTU;
|
||||
}
|
||||
|
||||
let mut config = VirtioNetConfig::default();
|
||||
build_net_config_space(&mut config, mac_addr, num_queues, mtu, &mut avail_features);
|
||||
let mut config = VirtioNetConfig::default();
|
||||
build_net_config_space(&mut config, mac_addr, num_queues, mtu, &mut avail_features);
|
||||
|
||||
let avail_protocol_features = VhostUserProtocolFeatures::MQ
|
||||
| VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS
|
||||
| VhostUserProtocolFeatures::REPLY_ACK
|
||||
| VhostUserProtocolFeatures::INFLIGHT_SHMFD
|
||||
| VhostUserProtocolFeatures::LOG_SHMFD;
|
||||
let avail_protocol_features = VhostUserProtocolFeatures::MQ
|
||||
| VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS
|
||||
| VhostUserProtocolFeatures::REPLY_ACK
|
||||
| VhostUserProtocolFeatures::INFLIGHT_SHMFD
|
||||
| VhostUserProtocolFeatures::LOG_SHMFD;
|
||||
|
||||
let (mut acked_features, acked_protocol_features) =
|
||||
vu.negotiate_features_vhost_user(avail_features, avail_protocol_features)?;
|
||||
let (mut acked_features, acked_protocol_features) =
|
||||
vu.negotiate_features_vhost_user(avail_features, avail_protocol_features)?;
|
||||
|
||||
let backend_num_queues =
|
||||
if acked_protocol_features & VhostUserProtocolFeatures::MQ.bits() != 0 {
|
||||
vu.socket_handle()
|
||||
.get_queue_num()
|
||||
.map_err(Error::VhostUserGetQueueMaxNum)?
|
||||
as usize
|
||||
} else {
|
||||
DEFAULT_QUEUE_NUMBER
|
||||
};
|
||||
let backend_num_queues =
|
||||
if acked_protocol_features & VhostUserProtocolFeatures::MQ.bits() != 0 {
|
||||
vu.socket_handle()
|
||||
.get_queue_num()
|
||||
.map_err(Error::VhostUserGetQueueMaxNum)? as usize
|
||||
} else {
|
||||
DEFAULT_QUEUE_NUMBER
|
||||
};
|
||||
|
||||
if num_queues > backend_num_queues {
|
||||
error!("vhost-user-net requested too many queues ({}) since the backend only supports {}\n",
|
||||
if num_queues > backend_num_queues {
|
||||
error!("vhost-user-net requested too many queues ({}) since the backend only supports {}\n",
|
||||
num_queues, backend_num_queues);
|
||||
return Err(Error::BadQueueNum);
|
||||
}
|
||||
return Err(Error::BadQueueNum);
|
||||
}
|
||||
|
||||
// If the control queue feature has been negotiated, let's increase
|
||||
// the number of queues.
|
||||
let vu_num_queues = num_queues;
|
||||
if acked_features & (1 << VIRTIO_NET_F_CTRL_VQ) != 0 {
|
||||
num_queues += 1;
|
||||
}
|
||||
// If the control queue feature has been negotiated, let's increase
|
||||
// the number of queues.
|
||||
let vu_num_queues = num_queues;
|
||||
if acked_features & (1 << VIRTIO_NET_F_CTRL_VQ) != 0 {
|
||||
num_queues += 1;
|
||||
}
|
||||
|
||||
// Make sure the virtio feature to set the MAC address is exposed to
|
||||
// the guest, even if it hasn't been negotiated with the backend.
|
||||
acked_features |= 1 << VIRTIO_NET_F_MAC;
|
||||
// Make sure the virtio feature to set the MAC address is exposed to
|
||||
// the guest, even if it hasn't been negotiated with the backend.
|
||||
acked_features |= 1 << VIRTIO_NET_F_MAC;
|
||||
|
||||
(
|
||||
acked_features,
|
||||
// If part of the available features that have been acked,
|
||||
// the PROTOCOL_FEATURES bit must be already set through
|
||||
// the VIRTIO acked features as we know the guest would
|
||||
// never ack it, thus the feature would be lost.
|
||||
acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(),
|
||||
acked_protocol_features,
|
||||
vu_num_queues,
|
||||
config,
|
||||
)
|
||||
};
|
||||
(
|
||||
acked_features,
|
||||
// If part of the available features that have been acked,
|
||||
// the PROTOCOL_FEATURES bit must be already set through
|
||||
// the VIRTIO acked features as we know the guest would
|
||||
// never ack it, thus the feature would be lost.
|
||||
acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(),
|
||||
acked_protocol_features,
|
||||
vu_num_queues,
|
||||
config,
|
||||
false,
|
||||
)
|
||||
};
|
||||
|
||||
Ok(Net {
|
||||
id,
|
||||
@@ -193,6 +201,7 @@ impl Net {
|
||||
acked_features,
|
||||
paused_sync: Some(Arc::new(Barrier::new(2))),
|
||||
min_queues: DEFAULT_QUEUE_NUMBER as u16,
|
||||
paused: Arc::new(AtomicBool::new(paused)),
|
||||
..Default::default()
|
||||
},
|
||||
vu_common: VhostUserCommon {
|
||||
|
||||
Reference in New Issue
Block a user