vmm: Move codebase to GuestMemoryAtomic from vm-memory

Relying on the latest vm-memory version, including the freshly
introduced structure GuestMemoryAtomic, this patch replaces every
occurrence of Arc<ArcSwap<GuestMemoryMmap> with
GuestMemoryAtomic<GuestMemoryMmap>.

The point is to rely on the common RCU-like implementation from
vm-memory so that we don't have to do it from Cloud-Hypervisor.

Fixes #735

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

vmm/src/cpu.rs

@@ -11,7 +11,6 @@
 use crate::device_manager::DeviceManager;
 #[cfg(feature = "acpi")]
 use acpi_tables::{aml, aml::Aml, sdt::SDT};
-use arc_swap::ArcSwap;
 #[cfg(feature = "acpi")]
 use arch::layout;
 use devices::{ioapic, BusDevice};
@@ -25,7 +24,7 @@ use std::sync::{Arc, Barrier, Mutex, Weak};
 use std::thread;
 use std::{fmt, io, result};
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
-use vm_memory::{Address, GuestAddress, GuestMemoryMmap};
+use vm_memory::{Address, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::signal::{register_signal_handler, SIGRTMIN};
 
@@ -277,7 +276,7 @@ impl Vcpu {
     pub fn configure(
         &mut self,
         kernel_start_addr: Option<GuestAddress>,
-        vm_memory: &Arc<ArcSwap<GuestMemoryMmap>>,
+        vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
         cpuid: CpuId,
     ) -> Result<()> {
         let mut cpuid = cpuid;
@@ -297,7 +296,7 @@ impl Vcpu {
             )
             .map_err(Error::REGSConfiguration)?;
             arch::x86_64::regs::setup_fpu(&self.fd).map_err(Error::FPUConfiguration)?;
-            arch::x86_64::regs::setup_sregs(&vm_memory.load(), &self.fd)
+            arch::x86_64::regs::setup_sregs(&vm_memory.memory(), &self.fd)
                 .map_err(Error::SREGSConfiguration)?;
         }
         arch::x86_64::interrupts::set_lint(&self.fd).map_err(Error::LocalIntConfiguration)?;
@@ -376,7 +375,7 @@ pub struct CpuManager {
     io_bus: Weak<devices::Bus>,
     mmio_bus: Arc<devices::Bus>,
     ioapic: Option<Arc<Mutex<ioapic::Ioapic>>>,
-    vm_memory: Arc<ArcSwap<GuestMemoryMmap>>,
+    vm_memory: GuestMemoryAtomic<GuestMemoryMmap>,
     cpuid: CpuId,
     fd: Arc<VmFd>,
     vcpus_kill_signalled: Arc<AtomicBool>,
@@ -496,7 +495,7 @@ impl CpuManager {
         boot_vcpus: u8,
         max_vcpus: u8,
         device_manager: &DeviceManager,
-        guest_memory: Arc<ArcSwap<GuestMemoryMmap>>,
+        guest_memory: GuestMemoryAtomic<GuestMemoryMmap>,
         fd: Arc<VmFd>,
         cpuid: CpuId,
         reset_evt: EventFd,

vmm/src/device_manager.rs

@@ -49,6 +49,8 @@ use vm_device::interrupt::{
 };
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::guest_memory::FileOffset;
+#[cfg(feature = "cmos")]
+use vm_memory::GuestAddressSpace;
 use vm_memory::{Address, GuestAddress, GuestUsize, MmapRegion};
 #[cfg(feature = "pci_support")]
 use vm_virtio::transport::VirtioPciDevice;
@@ -739,7 +741,7 @@ impl DeviceManager {
                 .lock()
                 .unwrap()
                 .guest_memory()
-                .load()
+                .memory()
                 .last_addr()
                 .0
                 + 1;

vmm/src/memory_manager.rs

@@ -5,7 +5,6 @@
 
 #[cfg(feature = "acpi")]
 use acpi_tables::{aml, aml::Aml};
-use arc_swap::ArcSwap;
 use arch::RegionType;
 use devices::BusDevice;
 use kvm_bindings::kvm_userspace_memory_region;
@@ -19,8 +18,9 @@ use std::sync::{Arc, Mutex};
 use vm_allocator::SystemAllocator;
 use vm_memory::guest_memory::FileOffset;
 use vm_memory::{
-    mmap::MmapRegionError, Address, Error as MmapError, GuestAddress, GuestMemory, GuestMemoryMmap,
-    GuestMemoryRegion, GuestRegionMmap, GuestUsize, MmapRegion,
+    mmap::MmapRegionError, Address, Error as MmapError, GuestAddress, GuestAddressSpace,
+    GuestMemory, GuestMemoryAtomic, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap,
+    GuestUsize, MmapRegion,
 };
 
 const HOTPLUG_COUNT: usize = 8;
@@ -35,12 +35,11 @@ struct HotPlugState {
 }
 
 pub struct MemoryManager {
-    guest_memory: Arc<ArcSwap<GuestMemoryMmap>>,
+    guest_memory: GuestMemoryAtomic<GuestMemoryMmap>,
     next_kvm_memory_slot: u32,
     start_of_device_area: GuestAddress,
     end_of_device_area: GuestAddress,
     fd: Arc<VmFd>,
-    mem_regions: Vec<Arc<GuestRegionMmap>>,
     hotplug_slots: Vec<HotPlugState>,
     selected_slot: usize,
     backing_file: Option<PathBuf>,
@@ -219,7 +218,7 @@ impl MemoryManager {
         }
 
         let guest_memory =
-            GuestMemoryMmap::from_arc_regions(mem_regions.clone()).map_err(Error::GuestMemory)?;
+            GuestMemoryMmap::from_arc_regions(mem_regions).map_err(Error::GuestMemory)?;
 
         let end_of_device_area = GuestAddress((1 << get_host_cpu_phys_bits()) - 1);
         let mem_end = guest_memory.last_addr();
@@ -233,7 +232,7 @@ impl MemoryManager {
             start_of_device_area = start_of_device_area.unchecked_add(size);
         }
 
-        let guest_memory = Arc::new(ArcSwap::new(Arc::new(guest_memory)));
+        let guest_memory = GuestMemoryAtomic::new(guest_memory);
 
         let mut hotplug_slots = Vec::with_capacity(HOTPLUG_COUNT);
         hotplug_slots.resize_with(HOTPLUG_COUNT, HotPlugState::default);
@@ -244,7 +243,6 @@ impl MemoryManager {
             start_of_device_area,
             end_of_device_area,
             fd,
-            mem_regions,
             hotplug_slots,
             selected_slot: 0,
             backing_file: backing_file.clone(),
@@ -254,7 +252,7 @@ impl MemoryManager {
             next_hotplug_slot: 0,
         }));
 
-        guest_memory.load().with_regions(|_, region| {
+        guest_memory.memory().with_regions(|_, region| {
             let _ = memory_manager.lock().unwrap().create_userspace_mapping(
                 region.start_addr().raw_value(),
                 region.len() as u64,
@@ -331,7 +329,7 @@ impl MemoryManager {
 
         // Start address needs to be non-contiguous with last memory added (leaving a gap of 256MiB)
         // and also aligned to 128MiB boundary. It must also start at the 64bit start.
-        let mem_end = self.guest_memory.load().last_addr();
+        let mem_end = self.guest_memory.memory().last_addr();
         let start_addr = if mem_end < arch::layout::MEM_32BIT_RESERVED_START {
             arch::layout::RAM_64BIT_START
         } else {
@@ -371,15 +369,17 @@ impl MemoryManager {
         self.next_hotplug_slot += 1;
 
         // Update the GuestMemoryMmap with the new range
-        self.mem_regions.push(region);
-        let guest_memory = GuestMemoryMmap::from_arc_regions(self.mem_regions.clone())
+        let guest_memory = self
+            .guest_memory
+            .memory()
+            .insert_region(region)
             .map_err(Error::GuestMemory)?;
-        self.guest_memory.store(Arc::new(guest_memory));
+        self.guest_memory.lock().unwrap().replace(guest_memory);
 
         Ok(())
     }
 
-    pub fn guest_memory(&self) -> Arc<ArcSwap<GuestMemoryMmap>> {
+    pub fn guest_memory(&self) -> GuestMemoryAtomic<GuestMemoryMmap> {
         self.guest_memory.clone()
     }
 

vmm/src/vm.rs

@@ -39,13 +39,15 @@ use signal_hook::{iterator::Signals, SIGINT, SIGTERM, SIGWINCH};
 use std::ffi::CString;
 use std::fs::File;
 use std::io;
+use std::ops::Deref;
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, RwLock};
 use std::{result, str, thread};
 use vm_allocator::{GsiApic, SystemAllocator};
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{
-    Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestUsize,
+    Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryMmap,
+    GuestMemoryRegion, GuestUsize,
 };
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::terminal::Terminal;
@@ -388,9 +390,9 @@ impl Vm {
 
         let cmdline_cstring = CString::new(cmdline).map_err(Error::CmdLineCString)?;
         let guest_memory = self.memory_manager.lock().as_ref().unwrap().guest_memory();
-        let mem = guest_memory.load_full();
+        let mem = guest_memory.memory();
         let entry_addr = match linux_loader::loader::Elf::load(
-            mem.as_ref(),
+            mem.deref(),
             None,
             &mut self.kernel,
             Some(arch::layout::HIGH_RAM_START),
@@ -398,7 +400,7 @@ impl Vm {
             Ok(entry_addr) => entry_addr,
             Err(linux_loader::loader::Error::InvalidElfMagicNumber) => {
                 linux_loader::loader::BzImage::load(
-                    mem.as_ref(),
+                    mem.deref(),
                     None,
                     &mut self.kernel,
                     Some(arch::layout::HIGH_RAM_START),
@@ -409,7 +411,7 @@ impl Vm {
         };
 
         linux_loader::loader::load_cmdline(
-            mem.as_ref(),
+            mem.deref(),
             arch::layout::CMDLINE_START,
             &cmdline_cstring,
         )
@@ -423,7 +425,7 @@ impl Vm {
         #[cfg(feature = "acpi")]
         {
             rsdp_addr = Some(crate::acpi::create_acpi_tables(
-                &mem,
+                mem.deref(),
                 &self.devices,
                 &self.cpu_manager,
                 &self.memory_manager,