arch, pci, vmm: Initial switch to the hypervisor crate

Start moving the vmm, arch and pci crates to being hypervisor agnostic
by using the hypervisor trait and abstractions. This is not a complete
switch and there are still some remaining KVM dependencies.

Signed-off-by: Muminul Islam <muislam@microsoft.com>
Signed-off-by: Samuel Ortiz <sameo@linux.intel.com>

vmm/src/cpu.rs

@@ -24,15 +24,13 @@ use arch::EntryPoint;
 use arch::{CpuidPatch, CpuidReg};
 use devices::{interrupt_controller::InterruptController, BusDevice};
 #[cfg(target_arch = "aarch64")]
-use kvm_bindings::KVM_SYSTEM_EVENT_SHUTDOWN;
+use hypervisor::kvm::kvm_bindings::KVM_SYSTEM_EVENT_SHUTDOWN;
 #[cfg(target_arch = "x86_64")]
-use kvm_bindings::{
-    kvm_fpu, kvm_lapic_state, kvm_mp_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs,
-    kvm_xsave, CpuId, Msrs,
-};
-use kvm_ioctls::*;
+use hypervisor::CpuId;
+use hypervisor::VcpuExit;
+
 use libc::{c_void, siginfo_t};
-use serde_derive::{Deserialize, Serialize};
+
 #[cfg(target_arch = "x86_64")]
 use std::fmt;
 use std::os::unix::thread::JoinHandleExt;
@@ -103,23 +101,23 @@ impl fmt::Display for DebugIoPortRange {
 #[derive(Debug)]
 pub enum Error {
     /// Cannot open the VCPU file descriptor.
-    VcpuFd(kvm_ioctls::Error),
+    VcpuFd(anyhow::Error),
 
     /// Cannot run the VCPUs.
-    VcpuRun(kvm_ioctls::Error),
+    VcpuRun(anyhow::Error),
 
     /// Cannot spawn a new vCPU thread.
     VcpuSpawn(io::Error),
 
     /// Cannot patch the CPU ID
-    PatchCpuId(kvm_ioctls::Error),
+    PatchCpuId(anyhow::Error),
 
     /// The call to KVM_SET_CPUID2 failed.
-    SetSupportedCpusFailed(kvm_ioctls::Error),
+    SetSupportedCpusFailed(anyhow::Error),
 
     #[cfg(target_arch = "x86_64")]
     /// Cannot set the local interruption due to bad configuration.
-    LocalIntConfiguration(arch::x86_64::interrupts::Error),
+    LocalIntConfiguration(anyhow::Error),
 
     /// Error configuring VCPU
     VcpuConfiguration(arch::Error),
@@ -140,58 +138,58 @@ pub enum Error {
     DesiredVCPUCountExceedsMax,
 
     /// Failed to get KVM vcpu lapic.
-    VcpuGetLapic(kvm_ioctls::Error),
+    VcpuGetLapic(anyhow::Error),
 
     /// Failed to set KVM vcpu lapic.
-    VcpuSetLapic(kvm_ioctls::Error),
+    VcpuSetLapic(anyhow::Error),
 
     /// Failed to get KVM vcpu MP state.
-    VcpuGetMpState(kvm_ioctls::Error),
+    VcpuGetMpState(anyhow::Error),
 
     /// Failed to set KVM vcpu MP state.
-    VcpuSetMpState(kvm_ioctls::Error),
+    VcpuSetMpState(anyhow::Error),
 
     /// Failed to get KVM vcpu msrs.
-    VcpuGetMsrs(kvm_ioctls::Error),
+    VcpuGetMsrs(anyhow::Error),
 
     /// Failed to set KVM vcpu msrs.
-    VcpuSetMsrs(kvm_ioctls::Error),
+    VcpuSetMsrs(anyhow::Error),
 
     /// Failed to get KVM vcpu regs.
-    VcpuGetRegs(kvm_ioctls::Error),
+    VcpuGetRegs(anyhow::Error),
 
     /// Failed to set KVM vcpu regs.
-    VcpuSetRegs(kvm_ioctls::Error),
+    VcpuSetRegs(anyhow::Error),
 
     /// Failed to get KVM vcpu sregs.
-    VcpuGetSregs(kvm_ioctls::Error),
+    VcpuGetSregs(anyhow::Error),
 
     /// Failed to set KVM vcpu sregs.
-    VcpuSetSregs(kvm_ioctls::Error),
+    VcpuSetSregs(anyhow::Error),
 
     /// Failed to get KVM vcpu events.
-    VcpuGetVcpuEvents(kvm_ioctls::Error),
+    VcpuGetVcpuEvents(anyhow::Error),
 
     /// Failed to set KVM vcpu events.
-    VcpuSetVcpuEvents(kvm_ioctls::Error),
+    VcpuSetVcpuEvents(anyhow::Error),
 
     /// Failed to get KVM vcpu FPU.
-    VcpuGetFpu(kvm_ioctls::Error),
+    VcpuGetFpu(anyhow::Error),
 
     /// Failed to set KVM vcpu FPU.
-    VcpuSetFpu(kvm_ioctls::Error),
+    VcpuSetFpu(anyhow::Error),
 
     /// Failed to get KVM vcpu XSAVE.
-    VcpuGetXsave(kvm_ioctls::Error),
+    VcpuGetXsave(anyhow::Error),
 
     /// Failed to set KVM vcpu XSAVE.
-    VcpuSetXsave(kvm_ioctls::Error),
+    VcpuSetXsave(anyhow::Error),
 
     /// Failed to get KVM vcpu XCRS.
-    VcpuGetXcrs(kvm_ioctls::Error),
+    VcpuGetXcrs(anyhow::Error),
 
     /// Failed to set KVM vcpu XCRS.
-    VcpuSetXcrs(kvm_ioctls::Error),
+    VcpuSetXcrs(anyhow::Error),
 
     /// Error resuming vCPU on shutdown
     ResumeOnShutdown(MigratableError),
@@ -232,7 +230,7 @@ struct InterruptSourceOverride {
 
 /// A wrapper around creating and using a kvm-based VCPU.
 pub struct Vcpu {
-    fd: VcpuFd,
+    fd: Arc<dyn hypervisor::Vcpu>,
     id: u8,
     #[cfg(target_arch = "x86_64")]
     io_bus: Arc<devices::Bus>,
@@ -245,24 +243,6 @@ pub struct Vcpu {
     mpidr: u64,
 }
 
-#[cfg(target_arch = "x86_64")]
-#[derive(Clone, Serialize, Deserialize)]
-pub struct VcpuKvmState {
-    msrs: Msrs,
-    vcpu_events: kvm_vcpu_events,
-    regs: kvm_regs,
-    sregs: kvm_sregs,
-    fpu: kvm_fpu,
-    lapic_state: kvm_lapic_state,
-    xsave: kvm_xsave,
-    xcrs: kvm_xcrs,
-    mp_state: kvm_mp_state,
-}
-
-#[cfg(target_arch = "aarch64")]
-#[derive(Clone, Serialize, Deserialize)]
-pub struct VcpuKvmState {}
-
 impl Vcpu {
     /// Constructs a new VCPU for `vm`.
     ///
@@ -272,13 +252,13 @@ impl Vcpu {
     /// * `vm` - The virtual machine this vcpu will get attached to.
     pub fn new(
         id: u8,
-        fd: &Arc<VmFd>,
+        fd: &Arc<dyn hypervisor::Vm>,
         #[cfg(target_arch = "x86_64")] io_bus: Arc<devices::Bus>,
         mmio_bus: Arc<devices::Bus>,
         interrupt_controller: Option<Arc<Mutex<dyn InterruptController>>>,
         creation_ts: std::time::Instant,
     ) -> Result<Arc<Mutex<Self>>> {
-        let kvm_vcpu = fd.create_vcpu(id).map_err(Error::VcpuFd)?;
+        let kvm_vcpu = fd.create_vcpu(id).map_err(|e| Error::VcpuFd(e.into()))?;
         // Initially the cpuid per vCPU is the one supported by this VM.
         Ok(Arc::new(Mutex::new(Vcpu {
             fd: kvm_vcpu,
@@ -303,7 +283,7 @@ impl Vcpu {
     /// * `cpuid` - (x86_64) CpuId, wrapper over the `kvm_cpuid2` structure.
     pub fn configure(
         &mut self,
-        #[cfg(target_arch = "aarch64")] vm_fd: &VmFd,
+        #[cfg(target_arch = "aarch64")] vm_fd: &Arc<dyn hypervisor::Vm>,
         kernel_entry_point: Option<EntryPoint>,
         vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
         #[cfg(target_arch = "x86_64")] cpuid: CpuId,
@@ -413,75 +393,6 @@ impl Vcpu {
             ts.as_micros()
         );
     }
-
-    #[cfg(target_arch = "x86_64")]
-    fn kvm_state(&self) -> Result<VcpuKvmState> {
-        let mut msrs = arch::x86_64::regs::boot_msr_entries();
-        self.fd.get_msrs(&mut msrs).map_err(Error::VcpuGetMsrs)?;
-
-        let vcpu_events = self
-            .fd
-            .get_vcpu_events()
-            .map_err(Error::VcpuGetVcpuEvents)?;
-        let regs = self.fd.get_regs().map_err(Error::VcpuGetRegs)?;
-        let sregs = self.fd.get_sregs().map_err(Error::VcpuGetSregs)?;
-        let lapic_state = self.fd.get_lapic().map_err(Error::VcpuGetLapic)?;
-        let fpu = self.fd.get_fpu().map_err(Error::VcpuGetFpu)?;
-        let xsave = self.fd.get_xsave().map_err(Error::VcpuGetXsave)?;
-        let xcrs = self.fd.get_xcrs().map_err(Error::VcpuGetXsave)?;
-        let mp_state = self.fd.get_mp_state().map_err(Error::VcpuGetMpState)?;
-
-        Ok(VcpuKvmState {
-            msrs,
-            vcpu_events,
-            regs,
-            sregs,
-            fpu,
-            lapic_state,
-            xsave,
-            xcrs,
-            mp_state,
-        })
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn set_kvm_state(&mut self, state: &VcpuKvmState) -> Result<()> {
-        self.fd.set_regs(&state.regs).map_err(Error::VcpuSetRegs)?;
-
-        self.fd.set_fpu(&state.fpu).map_err(Error::VcpuSetFpu)?;
-
-        self.fd
-            .set_xsave(&state.xsave)
-            .map_err(Error::VcpuSetXsave)?;
-
-        self.fd
-            .set_sregs(&state.sregs)
-            .map_err(Error::VcpuSetSregs)?;
-
-        self.fd.set_xcrs(&state.xcrs).map_err(Error::VcpuSetXcrs)?;
-
-        self.fd.set_msrs(&state.msrs).map_err(Error::VcpuSetMsrs)?;
-
-        self.fd
-            .set_lapic(&state.lapic_state)
-            .map_err(Error::VcpuSetLapic)?;
-
-        self.fd
-            .set_mp_state(state.mp_state)
-            .map_err(Error::VcpuSetMpState)?;
-
-        Ok(())
-    }
-
-    #[cfg(target_arch = "aarch64")]
-    fn kvm_state(&self) -> Result<VcpuKvmState> {
-        unimplemented!();
-    }
-
-    #[cfg(target_arch = "aarch64")]
-    fn set_kvm_state(&mut self, _state: &VcpuKvmState) -> Result<()> {
-        Ok(())
-    }
 }
 
 const VCPU_SNAPSHOT_ID: &str = "vcpu";
@@ -492,7 +403,7 @@ impl Snapshottable for Vcpu {
     }
 
     fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot = serde_json::to_vec(&self.kvm_state().map_err(|e| {
+        let snapshot = serde_json::to_vec(&self.fd.cpu_state().map_err(|e| {
             MigratableError::Snapshot(anyhow!("Could not get vCPU KVM state {:?}", e))
         })?)
         .map_err(|e| MigratableError::Snapshot(e.into()))?;
@@ -521,7 +432,7 @@ impl Snapshottable for Vcpu {
                 }
             };
 
-            self.set_kvm_state(&vcpu_state).map_err(|e| {
+            self.fd.set_cpu_state(&vcpu_state).map_err(|e| {
                 MigratableError::Restore(anyhow!("Could not set the vCPU KVM state {:?}", e))
             })?;
 
@@ -547,7 +458,7 @@ pub struct CpuManager {
     #[cfg(target_arch = "x86_64")]
     cpuid: CpuId,
     #[cfg_attr(target_arch = "aarch64", allow(dead_code))]
-    fd: Arc<VmFd>,
+    fd: Arc<dyn hypervisor::Vm>,
     vcpus_kill_signalled: Arc<AtomicBool>,
     vcpus_pause_signalled: Arc<AtomicBool>,
     #[cfg_attr(target_arch = "aarch64", allow(dead_code))]
@@ -677,16 +588,16 @@ impl CpuManager {
         config: &CpusConfig,
         device_manager: &Arc<Mutex<DeviceManager>>,
         guest_memory: GuestMemoryAtomic<GuestMemoryMmap>,
-        #[cfg_attr(target_arch = "aarch64", allow(unused_variables))] kvm: &Kvm,
-        fd: Arc<VmFd>,
+        fd: Arc<dyn hypervisor::Vm>,
         reset_evt: EventFd,
+        hypervisor: Arc<dyn hypervisor::Hypervisor>,
     ) -> Result<Arc<Mutex<CpuManager>>> {
         let mut vcpu_states = Vec::with_capacity(usize::from(config.max_vcpus));
         vcpu_states.resize_with(usize::from(config.max_vcpus), VcpuState::default);
 
         let device_manager = device_manager.lock().unwrap();
         #[cfg(target_arch = "x86_64")]
-        let cpuid = CpuManager::patch_cpuid(kvm, &config.topology)?;
+        let cpuid = CpuManager::patch_cpuid(hypervisor, &config.topology)?;
         let cpu_manager = Arc::new(Mutex::new(CpuManager {
             config: config.clone(),
             #[cfg(target_arch = "x86_64")]
@@ -725,7 +636,10 @@ impl CpuManager {
     }
 
     #[cfg(target_arch = "x86_64")]
-    fn patch_cpuid(kvm: &Kvm, topology: &Option<CpuTopology>) -> Result<CpuId> {
+    fn patch_cpuid(
+        hypervisor: Arc<dyn hypervisor::Hypervisor>,
+        topology: &Option<CpuTopology>,
+    ) -> Result<CpuId> {
         let mut cpuid_patches = Vec::new();
 
         // Patch tsc deadline timer bit
@@ -751,9 +665,9 @@ impl CpuManager {
         });
 
         // Supported CPUID
-        let mut cpuid = kvm
-            .get_supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES)
-            .map_err(Error::PatchCpuId)?;
+        let mut cpuid = hypervisor
+            .get_cpuid()
+            .map_err(|e| Error::PatchCpuId(e.into()))?;
 
         CpuidPatch::patch_cpuid(&mut cpuid, cpuid_patches);
 
@@ -804,7 +718,7 @@ impl CpuManager {
                     .unwrap()
                     .fd
                     .set_cpuid2(&cpuid)
-                    .map_err(Error::SetSupportedCpusFailed)?;
+                    .map_err(|e| Error::SetSupportedCpusFailed(e.into()))?;
             }
             vcpu.lock()
                 .unwrap()

vmm/src/device_manager.rs

@@ -40,7 +40,10 @@ use devices::{
     interrupt_controller, interrupt_controller::InterruptController, legacy::Serial, BusDevice,
     HotPlugNotificationFlags,
 };
-use kvm_ioctls::*;
+use hypervisor::kvm_ioctls;
+use hypervisor::kvm_ioctls::*;
+#[cfg(feature = "mmio_support")]
+use hypervisor::vm::DataMatch;
 use libc::TIOCGWINSZ;
 use libc::{MAP_NORESERVE, MAP_PRIVATE, MAP_SHARED, O_TMPFILE, PROT_READ, PROT_WRITE};
 #[cfg(feature = "pci_support")]
@@ -186,10 +189,10 @@ pub enum DeviceManagerError {
     FreePciBars(pci::PciDeviceError),
 
     /// Cannot register ioevent.
-    RegisterIoevent(kvm_ioctls::Error),
+    RegisterIoevent(anyhow::Error),
 
     /// Cannot unregister ioevent.
-    UnRegisterIoevent(kvm_ioctls::Error),
+    UnRegisterIoevent(anyhow::Error),
 
     /// Cannot create virtio device
     VirtioDevice(vmm_sys_util::errno::Error),
@@ -229,7 +232,7 @@ pub enum DeviceManagerError {
     VfioMapRegion(pci::VfioPciError),
 
     /// Failed to create the KVM device.
-    CreateKvmDevice(kvm_ioctls::Error),
+    CreateKvmDevice(anyhow::Error),
 
     /// Failed to memory map.
     Mmap(io::Error),
@@ -430,7 +433,7 @@ struct AddressManager {
     #[cfg(target_arch = "x86_64")]
     io_bus: Arc<devices::Bus>,
     mmio_bus: Arc<devices::Bus>,
-    vm_fd: Arc<VmFd>,
+    vm_fd: Arc<dyn hypervisor::Vm>,
     #[cfg(feature = "pci_support")]
     device_tree: Arc<Mutex<DeviceTree>>,
 }
@@ -561,15 +564,16 @@ impl DeviceRelocation for AddressManager {
             if bar_addr == new_base {
                 for (event, addr) in virtio_pci_dev.ioeventfds(old_base) {
                     let io_addr = IoEventAddress::Mmio(addr);
-                    self.vm_fd
-                        .unregister_ioevent(event, &io_addr)
-                        .map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
+                    self.vm_fd.unregister_ioevent(event, &io_addr).unwrap();
+                    /*
+                    map_err(io::Error::from_raw_os_error(
+                        hypervisor::HypervisorVmError::UnregisterIoEvent,
+                    ))?; */
                 }
                 for (event, addr) in virtio_pci_dev.ioeventfds(new_base) {
                     let io_addr = IoEventAddress::Mmio(addr);
-                    self.vm_fd
-                        .register_ioevent(event, &io_addr, NoDatamatch)
-                        .map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
+                    self.vm_fd.register_ioevent(event, &io_addr, None).unwrap();
+                    //.map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
                 }
             } else {
                 let virtio_dev = virtio_pci_dev.virtio_device();
@@ -577,7 +581,7 @@ impl DeviceRelocation for AddressManager {
                 if let Some(mut shm_regions) = virtio_dev.get_shm_regions() {
                     if shm_regions.addr.raw_value() == old_base {
                         // Remove old region from KVM by passing a size of 0.
-                        let mut mem_region = kvm_bindings::kvm_userspace_memory_region {
+                        let mut mem_region = hypervisor::kvm::kvm_userspace_memory_region {
                             slot: shm_regions.mem_slot,
                             guest_phys_addr: old_base,
                             memory_size: 0,
@@ -585,23 +589,19 @@ impl DeviceRelocation for AddressManager {
                             flags: 0,
                         };
 
-                        // Safe because removing an existing guest region.
-                        unsafe {
-                            self.vm_fd
-                                .set_user_memory_region(mem_region)
-                                .map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
-                        }
+                        self.vm_fd.set_user_memory_region(mem_region).unwrap();
+                        //.map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
 
                         // Create new mapping by inserting new region to KVM.
                         mem_region.guest_phys_addr = new_base;
                         mem_region.memory_size = shm_regions.len;
 
-                        // Safe because the guest regions are guaranteed not to overlap.
-                        unsafe {
-                            self.vm_fd
-                                .set_user_memory_region(mem_region)
-                                .map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
-                        }
+                        self.vm_fd.set_user_memory_region(mem_region).map_err(|e| {
+                            io::Error::new(
+                                io::ErrorKind::Other,
+                                format!("failed to set user memory regions: {:?}", e),
+                            )
+                        })?;
 
                         // Update shared memory regions to reflect the new mapping.
                         shm_regions.addr = GuestAddress(new_base);
@@ -753,7 +753,7 @@ pub struct DeviceManager {
 
 impl DeviceManager {
     pub fn new(
-        vm_fd: Arc<VmFd>,
+        vm_fd: Arc<dyn hypervisor::Vm>,
         config: Arc<Mutex<VmConfig>>,
         memory_manager: Arc<Mutex<MemoryManager>>,
         _exit_evt: &EventFd,
@@ -2291,15 +2291,15 @@ impl DeviceManager {
     }
 
     #[cfg(feature = "pci_support")]
-    fn create_kvm_device(vm: &Arc<VmFd>) -> DeviceManagerResult<DeviceFd> {
-        let mut vfio_dev = kvm_bindings::kvm_create_device {
-            type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_VFIO,
+    fn create_kvm_device(vm: &Arc<dyn hypervisor::Vm>) -> DeviceManagerResult<DeviceFd> {
+        let mut vfio_dev = hypervisor::kvm::kvm_create_device {
+            type_: hypervisor::kvm::kvm_device_type_KVM_DEV_TYPE_VFIO,
             fd: 0,
             flags: 0,
         };
 
         vm.create_device(&mut vfio_dev)
-            .map_err(DeviceManagerError::CreateKvmDevice)
+            .map_err(|e| DeviceManagerError::CreateKvmDevice(e.into()))
     }
 
     #[cfg(not(feature = "pci_support"))]
@@ -2582,8 +2582,8 @@ impl DeviceManager {
             let io_addr = IoEventAddress::Mmio(addr);
             self.address_manager
                 .vm_fd
-                .register_ioevent(event, &io_addr, NoDatamatch)
-                .map_err(DeviceManagerError::RegisterIoevent)?;
+                .register_ioevent(event, &io_addr, None)
+                .map_err(|e| DeviceManagerError::RegisterIoevent(e.into()))?;
         }
 
         let virtio_pci_device = Arc::new(Mutex::new(virtio_pci_device));
@@ -2739,8 +2739,8 @@ impl DeviceManager {
             let io_addr = IoEventAddress::Mmio(*addr);
             self.address_manager
                 .vm_fd
-                .register_ioevent(event, &io_addr, i as u32)
-                .map_err(DeviceManagerError::RegisterIoevent)?;
+                .register_ioevent(event, &io_addr, Some(DataMatch::DataMatch32(i as u32)))
+                .map_err(|e| DeviceManagerError::RegisterIoevent(e.into()))?;
         }
 
         let interrupt_group = interrupt_manager
@@ -2979,7 +2979,7 @@ impl DeviceManager {
                     self.address_manager
                         .vm_fd
                         .unregister_ioevent(event, &io_addr)
-                        .map_err(DeviceManagerError::UnRegisterIoevent)?;
+                        .map_err(|e| DeviceManagerError::UnRegisterIoevent(e.into()))?;
                 }
 
                 (

vmm/src/interrupt.rs

@@ -4,8 +4,8 @@
 //
 
 use devices::interrupt_controller::InterruptController;
-use kvm_bindings::{kvm_irq_routing, kvm_irq_routing_entry, KVM_IRQ_ROUTING_MSI};
-use kvm_ioctls::VmFd;
+use hypervisor::kvm::{kvm_irq_routing, kvm_irq_routing_entry, KVM_IRQ_ROUTING_MSI};
+
 use std::collections::HashMap;
 use std::io;
 use std::mem::size_of;
@@ -70,7 +70,7 @@ impl InterruptRoute {
         })
     }
 
-    pub fn enable(&self, vm: &Arc<VmFd>) -> Result<()> {
+    pub fn enable(&self, vm: &Arc<dyn hypervisor::Vm>) -> Result<()> {
         if !self.registered.load(Ordering::SeqCst) {
             vm.register_irqfd(&self.irq_fd, self.gsi).map_err(|e| {
                 io::Error::new(
@@ -86,7 +86,7 @@ impl InterruptRoute {
         Ok(())
     }
 
-    pub fn disable(&self, vm: &Arc<VmFd>) -> Result<()> {
+    pub fn disable(&self, vm: &Arc<dyn hypervisor::Vm>) -> Result<()> {
         if self.registered.load(Ordering::SeqCst) {
             vm.unregister_irqfd(&self.irq_fd, self.gsi).map_err(|e| {
                 io::Error::new(
@@ -109,14 +109,14 @@ pub struct KvmRoutingEntry {
 }
 
 pub struct MsiInterruptGroup {
-    vm_fd: Arc<VmFd>,
+    vm_fd: Arc<dyn hypervisor::Vm>,
     gsi_msi_routes: Arc<Mutex<HashMap<u32, KvmRoutingEntry>>>,
     irq_routes: HashMap<InterruptIndex, InterruptRoute>,
 }
 
 impl MsiInterruptGroup {
     fn new(
-        vm_fd: Arc<VmFd>,
+        vm_fd: Arc<dyn hypervisor::Vm>,
         gsi_msi_routes: Arc<Mutex<HashMap<u32, KvmRoutingEntry>>>,
         irq_routes: HashMap<InterruptIndex, InterruptRoute>,
     ) -> Self {
@@ -316,7 +316,7 @@ pub struct KvmLegacyUserspaceInterruptManager {
 
 pub struct KvmMsiInterruptManager {
     allocator: Arc<Mutex<SystemAllocator>>,
-    vm_fd: Arc<VmFd>,
+    vm_fd: Arc<dyn hypervisor::Vm>,
     gsi_msi_routes: Arc<Mutex<HashMap<u32, KvmRoutingEntry>>>,
 }
 
@@ -329,7 +329,7 @@ impl KvmLegacyUserspaceInterruptManager {
 impl KvmMsiInterruptManager {
     pub fn new(
         allocator: Arc<Mutex<SystemAllocator>>,
-        vm_fd: Arc<VmFd>,
+        vm_fd: Arc<dyn hypervisor::Vm>,
         gsi_msi_routes: Arc<Mutex<HashMap<u32, KvmRoutingEntry>>>,
     ) -> Self {
         KvmMsiInterruptManager {

vmm/src/lib.rs

@@ -5,6 +5,7 @@
 
 extern crate anyhow;
 extern crate arc_swap;
+extern crate hypervisor;
 #[macro_use]
 extern crate lazy_static;
 #[macro_use]
@@ -213,6 +214,7 @@ pub fn start_vmm_thread(
     api_sender: Sender<ApiRequest>,
     api_receiver: Receiver<ApiRequest>,
     seccomp_level: &SeccompLevel,
+    hypervisor: Arc<dyn hypervisor::Hypervisor>,
 ) -> Result<thread::JoinHandle<Result<()>>> {
     let http_api_event = api_event.try_clone().map_err(Error::EventFdClone)?;
 
@@ -232,7 +234,7 @@ pub fn start_vmm_thread(
             // Apply seccomp filter for VMM thread.
             SeccompFilter::apply(vmm_seccomp_filter).map_err(Error::ApplySeccompFilter)?;
 
-            let mut vmm = Vmm::new(vmm_version.to_string(), api_event, vmm_path)?;
+            let mut vmm = Vmm::new(vmm_version.to_string(), api_event, vmm_path, hypervisor)?;
 
             vmm.control_loop(Arc::new(api_receiver))
         })
@@ -253,10 +255,16 @@ pub struct Vmm {
     vm: Option<Vm>,
     vm_config: Option<Arc<Mutex<VmConfig>>>,
     vmm_path: PathBuf,
+    hypervisor: Arc<dyn hypervisor::Hypervisor>,
 }
 
 impl Vmm {
-    fn new(vmm_version: String, api_evt: EventFd, vmm_path: PathBuf) -> Result<Self> {
+    fn new(
+        vmm_version: String,
+        api_evt: EventFd,
+        vmm_path: PathBuf,
+        hypervisor: Arc<dyn hypervisor::Hypervisor>,
+    ) -> Result<Self> {
         let mut epoll = EpollContext::new().map_err(Error::Epoll)?;
         let exit_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?;
         let reset_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?;
@@ -286,6 +294,7 @@ impl Vmm {
             vm: None,
             vm_config: None,
             vmm_path,
+            hypervisor,
         })
     }
 
@@ -301,6 +310,7 @@ impl Vmm {
                     exit_evt,
                     reset_evt,
                     self.vmm_path.clone(),
+                    self.hypervisor.clone(),
                 )?;
                 self.vm = Some(vm);
             }
@@ -370,6 +380,7 @@ impl Vmm {
             self.vmm_path.clone(),
             source_url,
             restore_cfg.prefault,
+            self.hypervisor.clone(),
         )?;
         self.vm = Some(vm);
 
@@ -413,7 +424,13 @@ impl Vmm {
             if self.reset_evt.read().is_ok() {
                 warn!("Spurious second reset event received. Ignoring.");
             }
-            self.vm = Some(Vm::new(config, exit_evt, reset_evt, self.vmm_path.clone())?);
+            self.vm = Some(Vm::new(
+                config,
+                exit_evt,
+                reset_evt,
+                self.vmm_path.clone(),
+                self.hypervisor.clone(),
+            )?);
         }
 
         // Then we start the new VM.

vmm/src/memory_manager.rs

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 //
-
+extern crate hypervisor;
 use crate::config::{HotplugMethod, MemoryConfig};
 use crate::MEMORY_MANAGER_SNAPSHOT_ID;
 #[cfg(feature = "acpi")]
@@ -12,8 +12,9 @@ use arch::{get_host_cpu_phys_bits, layout, RegionType};
 #[cfg(target_arch = "x86_64")]
 use devices::ioapic;
 use devices::BusDevice;
-use kvm_bindings::{kvm_userspace_memory_region, KVM_MEM_READONLY};
-use kvm_ioctls::*;
+
+use hypervisor::kvm::{kvm_userspace_memory_region, KVM_MEM_READONLY};
+
 use std::convert::TryInto;
 use std::ffi;
 use std::fs::{File, OpenOptions};
@@ -56,7 +57,7 @@ pub struct MemoryManager {
     next_kvm_memory_slot: u32,
     start_of_device_area: GuestAddress,
     end_of_device_area: GuestAddress,
-    pub fd: Arc<VmFd>,
+    pub fd: Arc<dyn hypervisor::Vm>,
     hotplug_slots: Vec<HotPlugState>,
     selected_slot: usize,
     backing_file: Option<PathBuf>,
@@ -103,7 +104,7 @@ pub enum Error {
     InvalidSize,
 
     /// Failed to set the user memory region.
-    SetUserMemoryRegion(kvm_ioctls::Error),
+    SetUserMemoryRegion(hypervisor::HypervisorVmError),
 
     /// Failed to EventFd.
     EventFdFail(io::Error),
@@ -214,7 +215,7 @@ impl BusDevice for MemoryManager {
 
 impl MemoryManager {
     pub fn new(
-        fd: Arc<VmFd>,
+        fd: Arc<dyn hypervisor::Vm>,
         config: &MemoryConfig,
         ext_regions: Option<Vec<MemoryRegion>>,
         prefault: bool,
@@ -381,7 +382,7 @@ impl MemoryManager {
 
     pub fn new_from_snapshot(
         snapshot: &Snapshot,
-        fd: Arc<VmFd>,
+        fd: Arc<dyn hypervisor::Vm>,
         config: &MemoryConfig,
         source_url: &str,
         prefault: bool,
@@ -685,8 +686,8 @@ impl MemoryManager {
             flags: if readonly { KVM_MEM_READONLY } else { 0 },
         };
 
-        // Safe because the guest regions are guaranteed not to overlap.
-        unsafe { self.fd.set_user_memory_region(mem_region) }
+        self.fd
+            .set_user_memory_region(mem_region)
             .map_err(Error::SetUserMemoryRegion)?;
 
         // Mark the pages as mergeable if explicitly asked for.
@@ -738,8 +739,8 @@ impl MemoryManager {
             flags: 0,
         };
 
-        // Safe to remove because we know the region exist.
-        unsafe { self.fd.set_user_memory_region(mem_region) }
+        self.fd
+            .set_user_memory_region(mem_region)
             .map_err(Error::SetUserMemoryRegion)?;
 
         // Mark the pages as unmergeable if there were previously marked as

vmm/src/vm.rs

@@ -14,7 +14,7 @@
 extern crate arch;
 extern crate devices;
 extern crate epoll;
-extern crate kvm_ioctls;
+extern crate hypervisor;
 extern crate libc;
 extern crate linux_loader;
 extern crate net_util;
@@ -38,17 +38,13 @@ use crate::{
 use anyhow::anyhow;
 #[cfg(target_arch = "x86_64")]
 use arch::BootProtocol;
-use arch::{check_required_kvm_extensions, EntryPoint};
-#[cfg(target_arch = "x86_64")]
-use devices::ioapic;
+use arch::EntryPoint;
 use devices::HotPlugNotificationFlags;
-#[cfg(target_arch = "x86_64")]
-use kvm_bindings::{kvm_enable_cap, KVM_CAP_SPLIT_IRQCHIP};
-use kvm_ioctls::*;
 use linux_loader::cmdline::Cmdline;
 #[cfg(target_arch = "x86_64")]
 use linux_loader::loader::elf::Error::InvalidElfMagicNumber;
 use linux_loader::loader::KernelLoader;
+
 use signal_hook::{iterator::Signals, SIGINT, SIGTERM, SIGWINCH};
 #[cfg(target_arch = "x86_64")]
 use std::convert::TryInto;
@@ -82,12 +78,6 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     VmFd(io::Error),
 
-    /// Cannot create the KVM instance
-    VmCreate(kvm_ioctls::Error),
-
-    /// Cannot set the VM up
-    VmSetup(kvm_ioctls::Error),
-
     /// Cannot open the kernel image
     KernelFile(io::Error),
 
@@ -144,9 +134,6 @@ pub enum Error {
     /// Failed to join on vCPU threads
     ThreadCleanup(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
 
-    /// Failed to create a new KVM instance
-    KvmNew(kvm_ioctls::Error),
-
     /// VM is not created
     VmNotCreated,
 
@@ -272,57 +259,14 @@ pub struct Vm {
 }
 
 impl Vm {
-    fn kvm_new() -> Result<(Kvm, Arc<VmFd>)> {
-        let kvm = Kvm::new().map_err(Error::KvmNew)?;
-
-        check_required_kvm_extensions(&kvm).expect("Missing KVM capabilities");
-
-        let fd: VmFd;
-        loop {
-            match kvm.create_vm() {
-                Ok(res) => fd = res,
-                Err(e) => {
-                    if e.errno() == libc::EINTR {
-                        // If the error returned is EINTR, which means the
-                        // ioctl has been interrupted, we have to retry as
-                        // this can't be considered as a regular error.
-                        continue;
-                    } else {
-                        return Err(Error::VmCreate(e));
-                    }
-                }
-            }
-            break;
-        }
-        let fd = Arc::new(fd);
-
-        // Set TSS
-        #[cfg(target_arch = "x86_64")]
-        fd.set_tss_address(arch::x86_64::layout::KVM_TSS_ADDRESS.raw_value() as usize)
-            .map_err(Error::VmSetup)?;
-
-        #[cfg(target_arch = "x86_64")]
-        {
-            // Create split irqchip
-            // Only the local APIC is emulated in kernel, both PICs and IOAPIC
-            // are not.
-            let mut cap: kvm_enable_cap = Default::default();
-            cap.cap = KVM_CAP_SPLIT_IRQCHIP;
-            cap.args[0] = ioapic::NUM_IOAPIC_PINS as u64;
-            fd.enable_cap(&cap).map_err(Error::VmSetup)?;
-        }
-
-        Ok((kvm, fd))
-    }
-
     fn new_from_memory_manager(
         config: Arc<Mutex<VmConfig>>,
         memory_manager: Arc<Mutex<MemoryManager>>,
-        fd: Arc<VmFd>,
-        kvm: Kvm,
+        fd: Arc<dyn hypervisor::Vm>,
         exit_evt: EventFd,
         reset_evt: EventFd,
         vmm_path: PathBuf,
+        hypervisor: Arc<dyn hypervisor::Hypervisor>,
     ) -> Result<Self> {
         config
             .lock()
@@ -344,9 +288,9 @@ impl Vm {
             &config.lock().unwrap().cpus.clone(),
             &device_manager,
             memory_manager.lock().unwrap().guest_memory(),
-            &kvm,
             fd,
             reset_evt,
+            hypervisor,
         )
         .map_err(Error::CpuManager)?;
 
@@ -382,8 +326,13 @@ impl Vm {
         exit_evt: EventFd,
         reset_evt: EventFd,
         vmm_path: PathBuf,
+        hypervisor: Arc<dyn hypervisor::Hypervisor>,
     ) -> Result<Self> {
-        let (kvm, fd) = Vm::kvm_new()?;
+        #[cfg(target_arch = "x86_64")]
+        hypervisor.check_required_extensions().unwrap();
+        let fd = hypervisor.create_vm().unwrap();
+        #[cfg(target_arch = "x86_64")]
+        fd.enable_split_irq().unwrap();
         let memory_manager = MemoryManager::new(
             fd.clone(),
             &config.lock().unwrap().memory.clone(),
@@ -396,10 +345,10 @@ impl Vm {
             config,
             memory_manager,
             fd,
-            kvm,
             exit_evt,
             reset_evt,
             vmm_path,
+            hypervisor,
         )?;
 
         // The device manager must create the devices from here as it is part
@@ -420,8 +369,13 @@ impl Vm {
         vmm_path: PathBuf,
         source_url: &str,
         prefault: bool,
+        hypervisor: Arc<dyn hypervisor::Hypervisor>,
     ) -> Result<Self> {
-        let (kvm, fd) = Vm::kvm_new()?;
+        #[cfg(target_arch = "x86_64")]
+        hypervisor.check_required_extensions().unwrap();
+        let fd = hypervisor.create_vm().unwrap();
+        #[cfg(target_arch = "x86_64")]
+        fd.enable_split_irq().unwrap();
         let config = vm_config_from_snapshot(snapshot).map_err(Error::Restore)?;
 
         let memory_manager = if let Some(memory_manager_snapshot) =
@@ -445,10 +399,10 @@ impl Vm {
             config,
             memory_manager,
             fd,
-            kvm,
             exit_evt,
             reset_evt,
             vmm_path,
+            hypervisor,
         )
     }
 
@@ -1432,9 +1386,8 @@ mod tests {
 #[cfg(target_arch = "x86_64")]
 #[test]
 pub fn test_vm() {
-    use kvm_bindings::kvm_userspace_memory_region;
+    use hypervisor::VcpuExit;
     use vm_memory::{GuestMemory, GuestMemoryRegion};
-
     // This example based on https://lwn.net/Articles/658511/
     let code = [
         0xba, 0xf8, 0x03, /* mov $0x3f8, %dx */
@@ -1450,11 +1403,12 @@ pub fn test_vm() {
     let load_addr = GuestAddress(0x1000);
     let mem = GuestMemoryMmap::from_ranges(&[(load_addr, mem_size)]).unwrap();
 
-    let kvm = Kvm::new().expect("new KVM instance creation failed");
-    let vm_fd = kvm.create_vm().expect("new VM fd creation failed");
+    let kvm = hypervisor::kvm::KvmHyperVisor::new().unwrap();
+    let hv: Arc<dyn hypervisor::Hypervisor> = Arc::new(kvm);
+    let vm_fd = hv.create_vm().expect("new VM fd creation failed");
 
     mem.with_regions(|index, region| {
-        let mem_region = kvm_userspace_memory_region {
+        let mem_region = hypervisor::kvm_userspace_memory_region {
             slot: index as u32,
             guest_phys_addr: region.start_addr().raw_value(),
             memory_size: region.len() as u64,
@@ -1463,7 +1417,7 @@ pub fn test_vm() {
         };
 
         // Safe because the guest regions are guaranteed not to overlap.
-        unsafe { vm_fd.set_user_memory_region(mem_region) }
+        vm_fd.set_user_memory_region(mem_region)
     })
     .expect("Cannot configure guest memory");
     mem.write_slice(&code, load_addr)