cloud-hypervisor/vmm/src/interrupt.rs

// Copyright © 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
//

use std::collections::HashMap;
use std::io;
use std::sync::{Arc, Mutex};

use devices::interrupt_controller::InterruptController;
use hypervisor::IrqRoutingEntry;
use vm_allocator::SystemAllocator;
use vm_device::interrupt::{
    InterruptIndex, InterruptManager, InterruptSourceConfig, InterruptSourceGroup,
    LegacyIrqGroupConfig, MsiIrqGroupConfig,
};
use vmm_sys_util::eventfd::EventFd;

/// Reuse std::io::Result to simplify interoperability among crates.
type Result<T> = std::io::Result<T>;

struct InterruptRoute {
    gsi: Option<u32>,
    irq_fd: Option<EventFd>,
    registered: bool,
}

impl InterruptRoute {
    fn new() -> Result<Self> {
        // The irq_fd must be created eagerly because external components
        // (say, VFIO) need the fd at device initialization time via notifier().
        Self::new_with_fd(Some(EventFd::new(libc::EFD_NONBLOCK)?))
    }

    fn new_with_fd(irq_fd: Option<EventFd>) -> Result<Self> {
        Ok(InterruptRoute {
            gsi: None,
            irq_fd,
            registered: false,
        })
    }

    fn allocate_gsi(&mut self, allocator: &mut SystemAllocator) -> Result<u32> {
        match self.gsi {
            Some(existing) => Ok(existing),
            None => {
                let new_gsi = allocator
                    .allocate_gsi()
                    .ok_or_else(|| io::Error::other("Failed allocating new GSI"))?;
                self.gsi = Some(new_gsi);
                Ok(new_gsi)
            }
        }
    }

    fn enable(&mut self, vm: &dyn hypervisor::Vm) -> Result<()> {
        let gsi = match self.gsi {
            Some(gsi) => gsi,
            // Do nothing if no GSI was ever allocated for this route, which means the interrupt is still masked.
            None => return Ok(()),
        };

        if !self.registered {
            if let Some(ref irq_fd) = self.irq_fd {
                vm.register_irqfd(irq_fd, gsi)
                    .map_err(|e| io::Error::other(format!("Failed registering irq_fd: {e}")))?;
            }

            // Update internals to track the irq_fd as "registered".
            self.registered = true;
        }

        Ok(())
    }

    fn disable(&mut self, vm: &dyn hypervisor::Vm) -> Result<()> {
        let gsi = match self.gsi {
            Some(gsi) => gsi,
            // Do nothing if no GSI was ever allocated for this route, which means the interrupt is still masked.
            None => return Ok(()),
        };

        if self.registered {
            if let Some(ref irq_fd) = self.irq_fd {
                vm.unregister_irqfd(irq_fd, gsi)
                    .map_err(|e| io::Error::other(format!("Failed unregistering irq_fd: {e}")))?;
            }

            // Update internals to track the irq_fd as "unregistered".
            self.registered = false;
        }

        Ok(())
    }

    fn trigger(&mut self) -> Result<()> {
        match self.irq_fd {
            Some(ref fd) => fd.write(1),
            None => Ok(()),
        }
    }

    fn notifier(&mut self) -> Option<EventFd> {
        Some(
            self.irq_fd
                .as_ref()?
                .try_clone()
                .expect("Failed cloning interrupt's EventFd"),
        )
    }

    // This is currently not used, but the upcoming vhost-guest feature
    // will use it. Use #[allow(dead_code)] to suppress a compiler
    // warning.
    #[allow(dead_code)]
    fn set_notifier(&mut self, eventfd: Option<EventFd>, vm: &dyn hypervisor::Vm) -> Result<()> {
        let old_irqfd = core::mem::replace(&mut self.irq_fd, eventfd);
        if self.registered {
            // A registered route must have a GSI allocated, since enable()
            // only sets registered=true after using a valid GSI.
            let gsi = self.gsi.expect("registered route has no GSI allocated");
            if let Some(ref irq_fd) = self.irq_fd {
                vm.register_irqfd(irq_fd, gsi)
                    .map_err(|e| io::Error::other(format!("Failed registering irq_fd: {e}")))?;
            }
            // If the irqfd cannot be unregistered, what to do?  Spin?
            // Returning an error isn't helpful as the new irqfd is already registered.
            if let Some(old_irq_fd) = old_irqfd {
                match vm.unregister_irqfd(&old_irq_fd, gsi) {
                    Ok(()) => {}
                    Err(e) => log::warn!("Failed unregistering old irqfd: {e}"),
                }
            }
        }
        Ok(())
    }
}

struct RoutingEntry {
    route: IrqRoutingEntry,
    masked: bool,
}

struct MsiInterruptGroup {
    vm: Arc<dyn hypervisor::Vm>,
    gsi_msi_routes: Arc<Mutex<HashMap<u32, RoutingEntry>>>,
    irq_routes: HashMap<InterruptIndex, Mutex<InterruptRoute>>,
    allocator: Arc<Mutex<SystemAllocator>>,
}

impl MsiInterruptGroup {
    fn new(
        vm: Arc<dyn hypervisor::Vm>,
        gsi_msi_routes: Arc<Mutex<HashMap<u32, RoutingEntry>>>,
        irq_routes: HashMap<InterruptIndex, Mutex<InterruptRoute>>,
        allocator: Arc<Mutex<SystemAllocator>>,
    ) -> Self {
        MsiInterruptGroup {
            vm,
            gsi_msi_routes,
            irq_routes,
            allocator,
        }
    }

    fn set_gsi_routes(&self, routes: &HashMap<u32, RoutingEntry>) -> Result<()> {
        let mut entry_vec: Vec<IrqRoutingEntry> = Vec::new();
        for (_, entry) in routes.iter() {
            if entry.masked {
                continue;
            }

            entry_vec.push(entry.route);
        }

        self.vm
            .set_gsi_routing(&entry_vec)
            .map_err(|e| io::Error::other(format!("Failed setting GSI routing: {e}")))
    }
}

impl InterruptSourceGroup for MsiInterruptGroup {
    fn enable(&self) -> Result<()> {
        for (_, route) in self.irq_routes.iter() {
            route.lock().unwrap().enable(self.vm.as_ref())?;
        }

        Ok(())
    }

    fn disable(&self) -> Result<()> {
        for (_, route) in self.irq_routes.iter() {
            route.lock().unwrap().disable(self.vm.as_ref())?;
        }

        Ok(())
    }

    fn trigger(&self, index: InterruptIndex) -> Result<()> {
        if let Some(route) = self.irq_routes.get(&index) {
            return route.lock().unwrap().trigger();
        }

        Err(io::Error::other(format!(
            "trigger: Invalid interrupt index {index}"
        )))
    }

    fn notifier(&self, index: InterruptIndex) -> Option<EventFd> {
        if let Some(route) = self.irq_routes.get(&index) {
            return route.lock().unwrap().notifier();
        }

        None
    }

    fn update(
        &self,
        index: InterruptIndex,
        config: InterruptSourceConfig,
        masked: bool,
        set_gsi: bool,
    ) -> Result<()> {
        if let Some(route) = self.irq_routes.get(&index) {
            let mut route = route.lock().unwrap();
            let gsi = if masked {
                match route.gsi {
                    Some(gsi) => gsi,
                    // No update needed if masked and no GSI was ever allocated
                    None => return Ok(()),
                }
            } else {
                // Allocate a GSI when the interrupt vector is first unmasked
                let mut allocator = self.allocator.lock().unwrap();
                route.allocate_gsi(&mut allocator)?
            };

            let entry = RoutingEntry {
                route: self.vm.make_routing_entry(gsi, &config),
                masked,
            };

            // When mask a msi irq, entry.masked is set to be true,
            // and the gsi will not be passed to KVM through KVM_SET_GSI_ROUTING.
            // So it's required to call disable() (which deassign KVM_IRQFD) before
            // set_gsi_routes() to avoid kernel panic (see #3827)
            if masked {
                route.disable(self.vm.as_ref())?;
            }

            let mut routes = self.gsi_msi_routes.lock().unwrap();
            routes.insert(gsi, entry);
            if set_gsi {
                self.set_gsi_routes(&routes)?;
            }

            // Assign KVM_IRQFD after KVM_SET_GSI_ROUTING to avoid
            // panic on kernel which not have commit a80ced6ea514
            // (KVM: SVM: fix panic on out-of-bounds guest IRQ).
            if !masked {
                route.enable(self.vm.as_ref())?;
            }

            return Ok(());
        }

        Err(io::Error::other(format!(
            "update: Invalid interrupt index {index}"
        )))
    }

    fn set_gsi(&self) -> Result<()> {
        let routes = self.gsi_msi_routes.lock().unwrap();
        self.set_gsi_routes(&routes)
    }

    fn set_notifier(
        &mut self,
        index: InterruptIndex,
        eventfd: Option<EventFd>,
        vm: &dyn hypervisor::Vm,
    ) -> Result<()> {
        if let Some(route) = self.irq_routes.get(&index) {
            return route.lock().unwrap().set_notifier(eventfd, vm);
        }

        Ok(())
    }
}

struct LegacyUserspaceInterruptGroup {
    ioapic: Arc<Mutex<dyn InterruptController>>,
    irq: u32,
}

impl LegacyUserspaceInterruptGroup {
    fn new(ioapic: Arc<Mutex<dyn InterruptController>>, irq: u32) -> Self {
        LegacyUserspaceInterruptGroup { ioapic, irq }
    }
}

impl InterruptSourceGroup for LegacyUserspaceInterruptGroup {
    fn trigger(&self, _index: InterruptIndex) -> Result<()> {
        self.ioapic
            .lock()
            .unwrap()
            .service_irq(self.irq as usize)
            .map_err(|e| io::Error::other(format!("failed to inject IRQ #{}: {e:?}", self.irq)))
    }

    fn update(
        &self,
        _index: InterruptIndex,
        _config: InterruptSourceConfig,
        _masked: bool,
        _set_gsi: bool,
    ) -> Result<()> {
        Ok(())
    }

    fn set_gsi(&self) -> Result<()> {
        Ok(())
    }

    fn notifier(&self, _index: InterruptIndex) -> Option<EventFd> {
        self.ioapic.lock().unwrap().notifier(self.irq as usize)
    }
}

pub struct LegacyUserspaceInterruptManager {
    ioapic: Arc<Mutex<dyn InterruptController>>,
}

pub struct MsiInterruptManager {
    allocator: Arc<Mutex<SystemAllocator>>,
    vm: Arc<dyn hypervisor::Vm>,
    gsi_msi_routes: Arc<Mutex<HashMap<u32, RoutingEntry>>>,
}

impl LegacyUserspaceInterruptManager {
    pub fn new(ioapic: Arc<Mutex<dyn InterruptController>>) -> Self {
        LegacyUserspaceInterruptManager { ioapic }
    }
}

impl MsiInterruptManager {
    pub fn new(allocator: Arc<Mutex<SystemAllocator>>, vm: Arc<dyn hypervisor::Vm>) -> Self {
        // Create a shared list of GSI that can be shared through all PCI
        // devices. This way, we can maintain the full list of used GSI,
        // preventing one device from overriding interrupts setting from
        // another one.
        let gsi_msi_routes = Arc::new(Mutex::new(HashMap::new()));

        MsiInterruptManager {
            allocator,
            vm,
            gsi_msi_routes,
        }
    }
}

impl InterruptManager for LegacyUserspaceInterruptManager {
    type GroupConfig = LegacyIrqGroupConfig;

    fn create_group(&self, config: Self::GroupConfig) -> Result<Arc<dyn InterruptSourceGroup>> {
        Ok(Arc::new(LegacyUserspaceInterruptGroup::new(
            self.ioapic.clone(),
            config.irq,
        )))
    }

    fn destroy_group(&self, _group: Arc<dyn InterruptSourceGroup>) -> Result<()> {
        Ok(())
    }
}

impl MsiInterruptManager {
    fn create_group_raw(
        &self,
        config: <Self as InterruptManager>::GroupConfig,
    ) -> Result<MsiInterruptGroup> {
        let mut irq_routes: HashMap<InterruptIndex, Mutex<InterruptRoute>> =
            HashMap::with_capacity(config.count as usize);
        for i in config.base..config.base + config.count {
            irq_routes.insert(i, Mutex::new(InterruptRoute::new()?));
        }

        Ok(MsiInterruptGroup::new(
            self.vm.clone(),
            self.gsi_msi_routes.clone(),
            irq_routes,
            self.allocator.clone(),
        ))
    }
}

impl InterruptManager for MsiInterruptManager {
    type GroupConfig = MsiIrqGroupConfig;

    fn create_group(&self, config: Self::GroupConfig) -> Result<Arc<dyn InterruptSourceGroup>> {
        let mut irq_routes: HashMap<InterruptIndex, Mutex<InterruptRoute>> =
            HashMap::with_capacity(config.count as usize);
        for i in config.base..config.base + config.count {
            irq_routes.insert(i, Mutex::new(InterruptRoute::new()?));
        }

        Ok(Arc::new(MsiInterruptGroup::new(
            self.vm.clone(),
            self.gsi_msi_routes.clone(),
            irq_routes,
            self.allocator.clone(),
        )))
    }

    fn create_group_mut(
        &self,
        config: Self::GroupConfig,
    ) -> vm_device::interrupt::Result<Arc<Mutex<dyn InterruptSourceGroup>>> {
        let r = self.create_group_raw(config)?;
        Ok(Arc::new(Mutex::new(r)))
    }

    fn destroy_group(&self, _group: Arc<dyn InterruptSourceGroup>) -> Result<()> {
        Ok(())
    }
}