build: Release v30.1 (bug fix release)

Signed-off-by: Bo Chen <chen.bo@intel.com>

Cargo.lock

@@ -179,7 +179,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cloud-hypervisor"
-version = "30.0.0"
+version = "30.1.0"
 dependencies = [
  "anyhow",
  "api_client",
@@ -545,8 +545,7 @@ dependencies = [
 [[package]]
 name = "kvm-ioctls"
 version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8f8dc9c1896e5f144ec5d07169bc29f39a047686d29585a91f30489abfaeb6b"
+source = "git+https://github.com/rust-vmm/kvm-ioctls?branch=main#23a3bb045a467e60bb00328a0b13cea13b5815d0"
 dependencies = [
  "kvm-bindings",
  "libc",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "30.0.0"
+version = "30.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"
@@ -52,6 +52,7 @@ vm-memory = "0.10.0"
 # List of patched crates
 [patch.crates-io]
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.6.0-tdx" }
+kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "main" }
 versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_derive", branch = "ch" }
 
 [dev-dependencies]

hypervisor/src/kvm/mod.rs

@@ -330,7 +330,7 @@ impl KvmVm {
         Ok(VfioDeviceFd::new_from_kvm(device_fd))
     }
     /// Checks if a particular `Cap` is available.
-    fn check_extension(&self, c: Cap) -> bool {
+    pub fn check_extension(&self, c: Cap) -> bool {
         self.fd.check_extension(c)
     }
 }

release-notes.md

@@ -1,3 +1,4 @@
+- [v30.1](#v301)
 - [v30.0](#v300)
     - [Command Line Changes for Reduced Binary Size](#command-line-changes-for-reduced-binary-size)
     - [Basic vfio-user Server Support](#basic-vfio-user-server-support)
@@ -268,6 +269,20 @@
     - [Unit testing](#unit-testing)
     - [Integration tests parallelization](#integration-tests-parallelization)
 
+# v30.1
+
+This is a bug fix release. The following issues have been addressed:
+
+* Ignore and warn TAP FDs sent via the HTTP request body (#5350)
+* Properly preserve and close valid FDs for TAP devices (#5373)
+* Only use `KVM_ARM_VCPU_PMU_V3` if available (#5360)
+* Only touch the tty flags if it's being used (#5343)
+* Fix seccomp filter lists for vhost-user devices (#5361)
+* Fix the offset setting while removing the entire mapping of
+  `vhost-user` FS client (#5235)
+* Fix the `ShutdownVmm` and `Shutdown` commands to call the correct API
+  endpoint (#5322)
+
 # v30.0
 
 This release has been tracked in our [roadmap

scripts/run_integration_tests_aarch64.sh

@@ -148,7 +148,7 @@ update_workloads() {
     popd
 
     # Download Cloud Hypervisor binary from its last stable release
-    LAST_RELEASE_VERSION="v26.0"
+    LAST_RELEASE_VERSION="v30.0"
     CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64"
     CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
     pushd $WORKLOADS_DIR

scripts/run_integration_tests_live_migration.sh

@@ -46,7 +46,7 @@ fi
 popd
 
 # Download Cloud Hypervisor binary from its last stable release
-LAST_RELEASE_VERSION="v26.0"
+LAST_RELEASE_VERSION="v30.0"
 CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static"
 CH_RELEASE_NAME="cloud-hypervisor-static"
 pushd $WORKLOADS_DIR

src/bin/ch-remote.rs

@@ -329,7 +329,8 @@ fn do_command(toplevel: &TopLevel) -> Result<(), Error> {
             simple_api_command(&mut socket, "PUT", "delete", None).map_err(Error::ApiClient)
         }
         SubCommandEnum::ShutdownVmm(_) => {
-            simple_api_command(&mut socket, "PUT", "shutdown-vmm", None).map_err(Error::ApiClient)
+            simple_api_full_command(&mut socket, "PUT", "vmm.shutdown", None)
+                .map_err(Error::ApiClient)
         }
         SubCommandEnum::Resume(_) => {
             simple_api_command(&mut socket, "PUT", "resume", None).map_err(Error::ApiClient)
@@ -353,8 +354,7 @@ fn do_command(toplevel: &TopLevel) -> Result<(), Error> {
             simple_api_full_command(&mut socket, "GET", "vmm.ping", None).map_err(Error::ApiClient)
         }
         SubCommandEnum::Shutdown(_) => {
-            simple_api_full_command(&mut socket, "PUT", "vmm.shutdown", None)
-                .map_err(Error::ApiClient)
+            simple_api_command(&mut socket, "PUT", "shutdown", None).map_err(Error::ApiClient)
         }
         SubCommandEnum::Resize(ref config) => {
             resize_api_command(&mut socket, config.cpus, &config.memory, &config.balloon)

src/main.rs

@@ -21,7 +21,6 @@ use thiserror::Error;
 use vmm::config;
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::signal::block_signal;
-use vmm_sys_util::terminal::Terminal;
 
 #[cfg(feature = "dhat-heap")]
 #[global_allocator]
@@ -586,14 +585,6 @@ fn main() {
         }
     };
 
-    // SAFETY: trivially safe
-    let on_tty = unsafe { libc::isatty(libc::STDIN_FILENO) } != 0;
-    if on_tty {
-        // Don't forget to set the terminal in canonical mode
-        // before to exit.
-        std::io::stdin().lock().set_canon_mode().unwrap();
-    }
-
     #[cfg(feature = "dhat-heap")]
     drop(_profiler);
 
@@ -731,6 +722,7 @@ mod unit_tests {
             gdb: false,
             platform: None,
             tpm: None,
+            preserved_fds: None,
         };
 
         assert_eq!(expected_vm_config, result_vm_config);

tests/integration.rs

@@ -6275,6 +6275,18 @@ mod common_parallel {
                     .unwrap_or_default(),
                 2
             );
+
+            guest.reboot_linux(0, None);
+
+            assert_eq!(
+                guest
+                    .ssh_command("ip -o link | wc -l")
+                    .unwrap()
+                    .trim()
+                    .parse::<u32>()
+                    .unwrap_or_default(),
+                2
+            );
         });
 
         let _ = child.kill();
@@ -9335,51 +9347,43 @@ mod live_migration {
         }
 
         #[test]
-        #[ignore]
         fn test_live_upgrade_basic() {
             _test_live_migration(true, false)
         }
 
         #[test]
-        #[ignore]
         fn test_live_upgrade_local() {
             _test_live_migration(true, true)
         }
 
         #[test]
-        #[ignore]
         #[cfg(not(feature = "mshv"))]
         fn test_live_upgrade_numa() {
             _test_live_migration_numa(true, false)
         }
 
         #[test]
-        #[ignore]
         #[cfg(not(feature = "mshv"))]
         fn test_live_upgrade_numa_local() {
             _test_live_migration_numa(true, true)
         }
 
         #[test]
-        #[ignore]
         fn test_live_upgrade_watchdog() {
             _test_live_migration_watchdog(true, false)
         }
 
         #[test]
-        #[ignore]
         fn test_live_upgrade_watchdog_local() {
             _test_live_migration_watchdog(true, true)
         }
 
         #[test]
-        #[ignore]
         fn test_live_upgrade_balloon() {
             _test_live_migration_balloon(true, false)
         }
 
         #[test]
-        #[ignore]
         fn test_live_upgrade_balloon_local() {
             _test_live_migration_balloon(true, true)
         }
@@ -9406,7 +9410,6 @@ mod live_migration {
         }
 
         #[test]
-        #[ignore]
         #[cfg(target_arch = "x86_64")]
         #[cfg(not(feature = "mshv"))]
         fn test_live_upgrade_ovs_dpdk() {
@@ -9414,7 +9417,6 @@ mod live_migration {
         }
 
         #[test]
-        #[ignore]
         #[cfg(target_arch = "x86_64")]
         #[cfg(not(feature = "mshv"))]
         fn test_live_upgrade_ovs_dpdk_local() {

virtio-devices/src/seccomp_filters.rs

@@ -151,6 +151,7 @@ fn virtio_rng_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
 
 fn virtio_vhost_fs_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
     vec![
+        (libc::SYS_clock_nanosleep, vec![]),
         (libc::SYS_connect, vec![]),
         (libc::SYS_nanosleep, vec![]),
         (libc::SYS_pread64, vec![]),
@@ -170,8 +171,11 @@ fn virtio_vhost_net_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
     vec![
         (libc::SYS_accept4, vec![]),
         (libc::SYS_bind, vec![]),
+        (libc::SYS_clock_nanosleep, vec![]),
+        (libc::SYS_connect, vec![]),
         (libc::SYS_getcwd, vec![]),
         (libc::SYS_listen, vec![]),
+        (libc::SYS_nanosleep, vec![]),
         (libc::SYS_recvmsg, vec![]),
         (libc::SYS_sendmsg, vec![]),
         (libc::SYS_sendto, vec![]),
@@ -184,7 +188,14 @@ fn virtio_vhost_net_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
 }
 
 fn virtio_vhost_block_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
-    vec![]
+    vec![
+        (libc::SYS_clock_nanosleep, vec![]),
+        (libc::SYS_connect, vec![]),
+        (libc::SYS_nanosleep, vec![]),
+        (libc::SYS_recvmsg, vec![]),
+        (libc::SYS_sendmsg, vec![]),
+        (libc::SYS_socket, vec![]),
+    ]
 }
 
 fn create_vsock_ioctl_seccomp_rule() -> Vec<SeccompRule> {

virtio-devices/src/vhost_user/fs.rs

@@ -119,7 +119,6 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {
         debug!("fs_slave_unmap");
 
         for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
-            let offset = fs.cache_offset[i];
             let mut len = fs.len[i];
 
             // Ignore if the length is 0.
@@ -129,9 +128,12 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {
 
             // Need to handle a special case where the slave ask for the unmapping
             // of the entire mapping.
-            if len == 0xffff_ffff_ffff_ffff {
+            let offset = if len == 0xffff_ffff_ffff_ffff {
                 len = self.cache_size;
-            }
+                0
+            } else {
+                fs.cache_offset[i]
+            };
 
             if !self.is_req_valid(offset, len) {
                 return Err(io::Error::from_raw_os_error(libc::EINVAL));

vm-migration/src/lib.rs

@@ -13,7 +13,7 @@ pub mod protocol;
 
 /// Global VMM version for versioning
 const MAJOR_VERSION: u16 = 30;
-const MINOR_VERSION: u16 = 0;
+const MINOR_VERSION: u16 = 1;
 const VMM_VERSION: u16 = MAJOR_VERSION << 12 | MINOR_VERSION & 0b1111;
 
 pub trait VersionMapped {

vmm/src/api/http_endpoint.rs

@@ -105,6 +105,10 @@ impl EndpointHandler for VmActionHandler {
                 ),
                 AddNet(_) => {
                     let mut net_cfg: NetConfig = serde_json::from_slice(body.raw())?;
+                    if net_cfg.fds.is_some() {
+                        warn!("Ignoring FDs sent via the HTTP request body");
+                        net_cfg.fds = None;
+                    }
                     // Update network config with optional files that might have
                     // been sent through control message.
                     if !files.is_empty() {

vmm/src/config.rs

@@ -1134,38 +1134,6 @@ impl NetConfig {
     }
 }
 
-impl Clone for NetConfig {
-    fn clone(&self) -> Self {
-        NetConfig {
-            tap: self.tap.clone(),
-            vhost_socket: self.vhost_socket.clone(),
-            id: self.id.clone(),
-            fds: self
-                .fds
-                .as_ref()
-                // SAFETY: We have been handed these FDs through the API
-                .map(|fds| fds.iter().map(|fd| unsafe { libc::dup(*fd) }).collect()),
-            ..*self
-        }
-    }
-}
-
-impl Drop for NetConfig {
-    fn drop(&mut self) {
-        if let Some(mut fds) = self.fds.take() {
-            for fd in fds.drain(..) {
-                // Skip reserved FDs
-                if fd <= 2 {
-                    continue;
-                }
-
-                // SAFETY: Safe as the fd was given to the config by the API
-                unsafe { libc::close(fd) };
-            }
-        }
-    }
-}
-
 impl RngConfig {
     pub fn parse(rng: &str) -> Result<Self> {
         let mut parser = OptionParser::new();
@@ -2125,23 +2093,83 @@ impl VmConfig {
             gdb,
             platform,
             tpm,
+            preserved_fds: None,
         };
         config.validate().map_err(Error::Validation)?;
         Ok(config)
     }
 
+    /// # Safety
+    /// To use this safely, the caller must guarantee that the input
+    /// fds are all valid.
+    pub unsafe fn add_preserved_fds(&mut self, mut fds: Vec<i32>) {
+        if fds.is_empty() {
+            return;
+        }
+
+        if let Some(preserved_fds) = &self.preserved_fds {
+            fds.append(&mut preserved_fds.clone());
+        }
+
+        self.preserved_fds = Some(fds);
+    }
+
     #[cfg(feature = "tdx")]
     pub fn is_tdx_enabled(&self) -> bool {
         self.platform.as_ref().map(|p| p.tdx).unwrap_or(false)
     }
 }
 
+impl Clone for VmConfig {
+    fn clone(&self) -> Self {
+        VmConfig {
+            cpus: self.cpus.clone(),
+            memory: self.memory.clone(),
+            payload: self.payload.clone(),
+            disks: self.disks.clone(),
+            net: self.net.clone(),
+            rng: self.rng.clone(),
+            balloon: self.balloon.clone(),
+            fs: self.fs.clone(),
+            pmem: self.pmem.clone(),
+            serial: self.serial.clone(),
+            console: self.console.clone(),
+            devices: self.devices.clone(),
+            user_devices: self.user_devices.clone(),
+            vdpa: self.vdpa.clone(),
+            vsock: self.vsock.clone(),
+            #[cfg(target_arch = "x86_64")]
+            sgx_epc: self.sgx_epc.clone(),
+            numa: self.numa.clone(),
+            platform: self.platform.clone(),
+            tpm: self.tpm.clone(),
+            preserved_fds: self
+                .preserved_fds
+                .as_ref()
+                // SAFETY: FFI call with valid FDs
+                .map(|fds| fds.iter().map(|fd| unsafe { libc::dup(*fd) }).collect()),
+            ..*self
+        }
+    }
+}
+
+impl Drop for VmConfig {
+    fn drop(&mut self) {
+        if let Some(mut fds) = self.preserved_fds.take() {
+            for fd in fds.drain(..) {
+                // SAFETY: FFI call with valid FDs
+                unsafe { libc::close(fd) };
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use std::{fs::File, os::fd::AsRawFd};
-
     use super::*;
     use net_util::MacAddr;
+    use std::fs::File;
+    use std::os::unix::io::AsRawFd;
 
     #[test]
     fn test_cpu_parsing() -> Result<()> {
@@ -2361,10 +2389,6 @@ mod tests {
             NetConfig {
                 mac: MacAddr::parse_str("de:ad:be:ef:12:34").unwrap(),
                 host_mac: Some(MacAddr::parse_str("12:34:de:ad:be:ef").unwrap()),
-                fds: None,
-                id: None,
-                tap: None,
-                vhost_socket: None,
                 ..Default::default()
             }
         );
@@ -2375,9 +2399,6 @@ mod tests {
                 mac: MacAddr::parse_str("de:ad:be:ef:12:34").unwrap(),
                 host_mac: Some(MacAddr::parse_str("12:34:de:ad:be:ef").unwrap()),
                 id: Some("mynet0".to_owned()),
-                fds: None,
-                tap: None,
-                vhost_socket: None,
                 ..Default::default()
             }
         );
@@ -2392,9 +2413,6 @@ mod tests {
                 tap: Some("tap0".to_owned()),
                 ip: "192.168.100.1".parse().unwrap(),
                 mask: "255.255.255.128".parse().unwrap(),
-                fds: None,
-                id: None,
-                vhost_socket: None,
                 ..Default::default()
             }
         );
@@ -2408,9 +2426,6 @@ mod tests {
                 host_mac: Some(MacAddr::parse_str("12:34:de:ad:be:ef").unwrap()),
                 vhost_user: true,
                 vhost_socket: Some("/tmp/sock".to_owned()),
-                fds: None,
-                id: None,
-                tap: None,
                 ..Default::default()
             }
         );
@@ -2423,31 +2438,18 @@ mod tests {
                 num_queues: 4,
                 queue_size: 1024,
                 iommu: true,
-                fds: None,
-                id: None,
-                tap: None,
-                vhost_socket: None,
                 ..Default::default()
             }
         );
 
-        // SAFETY: Safe as the file was just opened
-        let fd1 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };
-        // SAFETY: Safe as the file was just opened
-        let fd2 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };
-
         assert_eq!(
-            &format!(
-                "{:?}",
-                NetConfig::parse(&format!(
-                    "mac=de:ad:be:ef:12:34,fd=[{fd1},{fd2}],num_queues=4"
-                ))?
-            ),
-            &format!("NetConfig {{ tap: None, ip: 192.168.249.1, mask: 255.255.255.0, \
-                mac: MacAddr {{ bytes: [222, 173, 190, 239, 18, 52] }}, host_mac: None, mtu: None, \
-                iommu: false, num_queues: 4, queue_size: 256, vhost_user: false, vhost_socket: None, \
-                vhost_mode: Client, id: None, fds: Some([{fd1}, {fd2}]), \
-                rate_limiter_config: None, pci_segment: 0, offload_tso: true, offload_ufo: true, offload_csum: true }}")
+            NetConfig::parse("mac=de:ad:be:ef:12:34,fd=[3,7],num_queues=4")?,
+            NetConfig {
+                mac: MacAddr::parse_str("de:ad:be:ef:12:34").unwrap(),
+                fds: Some(vec![3, 7]),
+                num_queues: 4,
+                ..Default::default()
+            }
         );
 
         Ok(())
@@ -2774,6 +2776,7 @@ mod tests {
             gdb: false,
             platform: None,
             tpm: None,
+            preserved_fds: None,
         };
 
         assert!(valid_config.validate().is_ok());
@@ -2868,10 +2871,6 @@ mod tests {
         let mut invalid_config = valid_config.clone();
         invalid_config.net = Some(vec![NetConfig {
             vhost_user: true,
-            fds: None,
-            id: None,
-            tap: None,
-            vhost_socket: None,
             ..Default::default()
         }]);
         assert_eq!(
@@ -2883,9 +2882,6 @@ mod tests {
         still_valid_config.net = Some(vec![NetConfig {
             vhost_user: true,
             vhost_socket: Some("/path/to/sock".to_owned()),
-            fds: None,
-            id: None,
-            tap: None,
             ..Default::default()
         }]);
         still_valid_config.memory.shared = true;
@@ -2894,9 +2890,6 @@ mod tests {
         let mut invalid_config = valid_config.clone();
         invalid_config.net = Some(vec![NetConfig {
             fds: Some(vec![0]),
-            id: None,
-            tap: None,
-            vhost_socket: None,
             ..Default::default()
         }]);
         assert_eq!(
@@ -2907,10 +2900,6 @@ mod tests {
         let mut invalid_config = valid_config.clone();
         invalid_config.net = Some(vec![NetConfig {
             offload_csum: false,
-            fds: None,
-            id: None,
-            tap: None,
-            vhost_socket: None,
             ..Default::default()
         }]);
         assert_eq!(
@@ -3014,10 +3003,6 @@ mod tests {
         still_valid_config.net = Some(vec![NetConfig {
             iommu: true,
             pci_segment: 1,
-            fds: None,
-            id: None,
-            tap: None,
-            vhost_socket: None,
             ..Default::default()
         }]);
         assert!(still_valid_config.validate().is_ok());
@@ -3086,10 +3071,6 @@ mod tests {
         invalid_config.net = Some(vec![NetConfig {
             iommu: false,
             pci_segment: 1,
-            fds: None,
-            id: None,
-            tap: None,
-            vhost_socket: None,
             ..Default::default()
         }]);
         assert_eq!(
@@ -3205,7 +3186,7 @@ mod tests {
         ]);
         assert!(still_valid_config.validate().is_ok());
 
-        let mut invalid_config = valid_config;
+        let mut invalid_config = valid_config.clone();
         invalid_config.devices = Some(vec![
             DeviceConfig {
                 path: "/device1".into(),
@@ -3217,5 +3198,16 @@ mod tests {
             },
         ]);
         assert!(invalid_config.validate().is_err());
+
+        let mut still_valid_config = valid_config;
+        // SAFETY: Safe as the file was just opened
+        let fd1 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };
+        // SAFETY: Safe as the file was just opened
+        let fd2 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };
+        // SAFETY: safe as both FDs are valid
+        unsafe {
+            still_valid_config.add_preserved_fds(vec![fd1, fd2]);
+        }
+        let _still_valid_config = still_valid_config.clone();
     }
 }

vmm/src/cpu.rs

@@ -52,6 +52,8 @@ use hypervisor::arch::x86::MsrEntry;
 use hypervisor::arch::x86::{SpecialRegisters, StandardRegisters};
 #[cfg(target_arch = "aarch64")]
 use hypervisor::kvm::kvm_bindings;
+#[cfg(all(target_arch = "aarch64", feature = "kvm"))]
+use hypervisor::kvm::kvm_ioctls::Cap;
 #[cfg(feature = "tdx")]
 use hypervisor::kvm::{TdxExitDetails, TdxExitStatus};
 use hypervisor::{CpuState, HypervisorCpuError, HypervisorType, VmExit, VmOps};
@@ -370,7 +372,14 @@ impl Vcpu {
             .map_err(Error::VcpuArmPreferredTarget)?;
         // We already checked that the capability is supported.
         kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PSCI_0_2;
-        kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3;
+        if vm
+            .as_any()
+            .downcast_ref::<hypervisor::kvm::KvmVm>()
+            .unwrap()
+            .check_extension(Cap::ArmPmuV3)
+        {
+            kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3;
+        }
         // Non-boot cpus are powered off initially.
         if self.id > 0 {
             kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;

vmm/src/device_manager.rs

@@ -70,6 +70,7 @@ use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::path::PathBuf;
 use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
 use tracer::trace_scoped;
@@ -100,6 +101,7 @@ use vm_migration::{
 use vm_virtio::AccessPlatform;
 use vm_virtio::VirtioDeviceType;
 use vmm_sys_util::eventfd::EventFd;
+use vmm_sys_util::terminal::Terminal;
 
 #[cfg(target_arch = "aarch64")]
 const MMIO_LEN: u64 = 0x1000;
@@ -823,6 +825,9 @@ pub struct DeviceManager {
     // pty foreground status,
     console_resize_pipe: Option<Arc<File>>,
 
+    // Are any devices using the tty?
+    on_tty: Option<Arc<AtomicBool>>,
+
     // Interrupt controller
     #[cfg(target_arch = "x86_64")]
     interrupt_controller: Option<Arc<Mutex<ioapic::Ioapic>>>,
@@ -1107,6 +1112,7 @@ impl DeviceManager {
             serial_manager: None,
             console_pty: None,
             console_resize_pipe: None,
+            on_tty: None,
             virtio_mem_devices: Vec::new(),
             #[cfg(target_arch = "aarch64")]
             gpio_device: None,
@@ -1154,6 +1160,7 @@ impl DeviceManager {
         serial_pty: Option<PtyPair>,
         console_pty: Option<PtyPair>,
         console_resize_pipe: Option<File>,
+        on_tty: Arc<AtomicBool>,
     ) -> DeviceManagerResult<()> {
         trace_scoped!("create_devices");
 
@@ -1215,7 +1222,9 @@ impl DeviceManager {
             serial_pty,
             console_pty,
             console_resize_pipe,
+            &on_tty,
         )?;
+        self.on_tty = Some(on_tty);
 
         if let Some(tpm) = self.config.clone().lock().unwrap().tpm.as_ref() {
             let tpm_dev = self.add_tpm_device(tpm.socket.clone())?;
@@ -1856,7 +1865,7 @@ impl DeviceManager {
         Ok(())
     }
 
-    fn set_raw_mode(&self, f: &mut File) -> vmm_sys_util::errno::Result<()> {
+    fn set_raw_mode(&self, f: &mut dyn AsRawFd) -> vmm_sys_util::errno::Result<()> {
         // SAFETY: FFI call. Variable t is guaranteed to be a valid termios from modify_mode.
         self.modify_mode(f.as_raw_fd(), |t| unsafe { cfmakeraw(t) })
     }
@@ -1886,6 +1895,7 @@ impl DeviceManager {
         virtio_devices: &mut Vec<MetaVirtioDevice>,
         console_pty: Option<PtyPair>,
         resize_pipe: Option<File>,
+        on_tty: &Arc<AtomicBool>,
     ) -> DeviceManagerResult<Option<Arc<virtio_devices::ConsoleResizer>>> {
         let console_config = self.config.lock().unwrap().console.clone();
         let endpoint = match console_config.mode {
@@ -1925,7 +1935,12 @@ impl DeviceManager {
                     return vmm_sys_util::errno::errno_result().map_err(DeviceManagerError::DupFd);
                 }
                 // SAFETY: stdout is valid and owned solely by us.
-                let stdout = unsafe { File::from_raw_fd(stdout) };
+                let mut stdout = unsafe { File::from_raw_fd(stdout) };
+
+                on_tty.store(true, Ordering::SeqCst);
+
+                // Make sure stdout is in raw mode, if it's a terminal.
+                let _ = self.set_raw_mode(&mut stdout);
 
                 // If an interactive TTY then we can accept input
                 // SAFETY: FFI call. Trivially safe.
@@ -1997,6 +2012,7 @@ impl DeviceManager {
         serial_pty: Option<PtyPair>,
         console_pty: Option<PtyPair>,
         console_resize_pipe: Option<File>,
+        on_tty: &Arc<AtomicBool>,
     ) -> DeviceManagerResult<Arc<Console>> {
         let serial_config = self.config.lock().unwrap().serial.clone();
         let serial_writer: Option<Box<dyn io::Write + Send>> = match serial_config.mode {
@@ -2018,7 +2034,12 @@ impl DeviceManager {
                 }
                 None
             }
-            ConsoleOutputMode::Tty => Some(Box::new(stdout())),
+            ConsoleOutputMode::Tty => {
+                let mut out = stdout();
+                on_tty.store(true, Ordering::SeqCst);
+                let _ = self.set_raw_mode(&mut out);
+                Some(Box::new(out))
+            }
             ConsoleOutputMode::Off | ConsoleOutputMode::Null => None,
         };
         if serial_config.mode != ConsoleOutputMode::Off {
@@ -2045,8 +2066,12 @@ impl DeviceManager {
             };
         }
 
-        let console_resizer =
-            self.add_virtio_console_device(virtio_devices, console_pty, console_resize_pipe)?;
+        let console_resizer = self.add_virtio_console_device(
+            virtio_devices,
+            console_pty,
+            console_resize_pipe,
+            on_tty,
+        )?;
 
         Ok(Arc::new(Console { console_resizer }))
     }
@@ -2381,26 +2406,31 @@ impl DeviceManager {
                     .map_err(DeviceManagerError::CreateVirtioNet)?,
                 ))
             } else if let Some(fds) = &net_cfg.fds {
-                Arc::new(Mutex::new(
-                    virtio_devices::Net::from_tap_fds(
-                        id.clone(),
-                        fds,
-                        Some(net_cfg.mac),
-                        net_cfg.mtu,
-                        self.force_iommu | net_cfg.iommu,
-                        net_cfg.queue_size,
-                        self.seccomp_action.clone(),
-                        net_cfg.rate_limiter_config,
-                        self.exit_evt
-                            .try_clone()
-                            .map_err(DeviceManagerError::EventFd)?,
-                        state,
-                        net_cfg.offload_tso,
-                        net_cfg.offload_ufo,
-                        net_cfg.offload_csum,
-                    )
-                    .map_err(DeviceManagerError::CreateVirtioNet)?,
-                ))
+                let net = virtio_devices::Net::from_tap_fds(
+                    id.clone(),
+                    fds,
+                    Some(net_cfg.mac),
+                    net_cfg.mtu,
+                    self.force_iommu | net_cfg.iommu,
+                    net_cfg.queue_size,
+                    self.seccomp_action.clone(),
+                    net_cfg.rate_limiter_config,
+                    self.exit_evt
+                        .try_clone()
+                        .map_err(DeviceManagerError::EventFd)?,
+                    state,
+                    net_cfg.offload_tso,
+                    net_cfg.offload_ufo,
+                    net_cfg.offload_csum,
+                )
+                .map_err(DeviceManagerError::CreateVirtioNet)?;
+
+                // SAFETY: 'fds' are valid because TAP devices are created successfully
+                unsafe {
+                    self.config.lock().unwrap().add_preserved_fds(fds.clone());
+                }
+
+                Arc::new(Mutex::new(net))
             } else {
                 Arc::new(Mutex::new(
                     virtio_devices::Net::new(
@@ -4625,5 +4655,11 @@ impl Drop for DeviceManager {
         for handle in self.virtio_devices.drain(..) {
             handle.virtio_device.lock().unwrap().shutdown();
         }
+
+        if let Some(ref on_tty) = self.on_tty {
+            if on_tty.load(Ordering::SeqCst) {
+                let _ = std::io::stdin().lock().set_canon_mode();
+            }
+        }
     }
 }

vmm/src/lib.rs

@@ -41,6 +41,7 @@ use std::os::unix::net::UnixListener;
 use std::os::unix::net::UnixStream;
 use std::panic::AssertUnwindSafe;
 use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{Receiver, RecvError, SendError, Sender};
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
@@ -407,12 +408,13 @@ pub struct Vmm {
     activate_evt: EventFd,
     signals: Option<Handle>,
     threads: Vec<thread::JoinHandle<()>>,
+    on_tty: Arc<AtomicBool>,
 }
 
 impl Vmm {
     pub const HANDLED_SIGNALS: [i32; 2] = [SIGTERM, SIGINT];
 
-    fn signal_handler(mut signals: Signals, on_tty: bool, exit_evt: &EventFd) {
+    fn signal_handler(mut signals: Signals, on_tty: Arc<AtomicBool>, exit_evt: &EventFd) {
         for sig in &Self::HANDLED_SIGNALS {
             unblock_signal(*sig).unwrap();
         }
@@ -422,7 +424,7 @@ impl Vmm {
                 SIGTERM | SIGINT => {
                     if exit_evt.write(1).is_err() {
                         // Resetting the terminal is usually done as the VMM exits
-                        if on_tty {
+                        if on_tty.load(Ordering::SeqCst) {
                             io::stdin()
                                 .lock()
                                 .set_canon_mode()
@@ -442,8 +444,7 @@ impl Vmm {
             Ok(signals) => {
                 self.signals = Some(signals.handle());
                 let exit_evt = self.exit_evt.try_clone().map_err(Error::EventFdClone)?;
-                // SAFETY: trivially safe
-                let on_tty = unsafe { libc::isatty(libc::STDIN_FILENO) } != 0;
+                let on_tty = Arc::clone(&self.on_tty);
 
                 let signal_handler_seccomp_filter = get_seccomp_filter(
                     &self.seccomp_action,
@@ -532,6 +533,7 @@ impl Vmm {
             activate_evt,
             signals: None,
             threads: vec![],
+            on_tty: Arc::new(AtomicBool::new(false)),
         })
     }
 
@@ -582,6 +584,7 @@ impl Vmm {
                         None,
                         None,
                         None,
+                        Arc::clone(&self.on_tty),
                         None,
                         None,
                         None,
@@ -680,6 +683,7 @@ impl Vmm {
             None,
             None,
             None,
+            Arc::clone(&self.on_tty),
             Some(snapshot),
             Some(source_url),
             Some(restore_cfg.prefault),
@@ -760,6 +764,7 @@ impl Vmm {
             serial_pty,
             console_pty,
             console_resize_pipe,
+            Arc::clone(&self.on_tty),
             None,
             None,
             None,
@@ -1262,6 +1267,7 @@ impl Vmm {
             None,
             None,
             None,
+            Arc::clone(&self.on_tty),
             Some(snapshot),
         )
         .map_err(|e| {
@@ -2130,6 +2136,7 @@ mod unit_tests {
             gdb: false,
             platform: None,
             tpm: None,
+            preserved_fds: None,
         }))
     }
 

vmm/src/pci_segment.rs

@@ -159,7 +159,7 @@ impl PciSegment {
 
         // There are 32 devices on the PCI bus, let's assign them an IRQ.
         for i in 0..32 {
-            pci_irq_slots[i] = irqs[(i % num_irqs)];
+            pci_irq_slots[i] = irqs[i % num_irqs];
         }
 
         Ok(())

vmm/src/vm.rs

@@ -81,6 +81,7 @@ use std::num::Wrapping;
 use std::ops::Deref;
 use std::os::unix::net::UnixStream;
 use std::panic::AssertUnwindSafe;
+use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::Instant;
 use std::{result, str, thread};
@@ -98,7 +99,6 @@ use vm_migration::{
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::signal::unblock_signal;
 use vmm_sys_util::sock_ctrl_msg::ScmSocket;
-use vmm_sys_util::terminal::Terminal;
 
 /// Errors associated with VM management
 #[derive(Debug, Error)]
@@ -141,12 +141,6 @@ pub enum Error {
     #[error("Error from device manager: {0:?}")]
     DeviceManager(DeviceManagerError),
 
-    #[error("Cannot setup terminal in raw mode: {0}")]
-    SetTerminalRaw(#[source] vmm_sys_util::errno::Error),
-
-    #[error("Cannot setup terminal in canonical mode.: {0}")]
-    SetTerminalCanon(#[source] vmm_sys_util::errno::Error),
-
     #[error("Cannot spawn a signal handler thread: {0}")]
     SignalHandlerSpawn(#[source] io::Error),
 
@@ -440,7 +434,6 @@ pub struct Vm {
     threads: Vec<thread::JoinHandle<()>>,
     device_manager: Arc<Mutex<DeviceManager>>,
     config: Arc<Mutex<VmConfig>>,
-    on_tty: bool,
     signals: Option<Handle>,
     state: RwLock<VmState>,
     cpu_manager: Arc<Mutex<cpu::CpuManager>>,
@@ -476,6 +469,7 @@ impl Vm {
         serial_pty: Option<PtyPair>,
         console_pty: Option<PtyPair>,
         console_resize_pipe: Option<File>,
+        on_tty: Arc<AtomicBool>,
         snapshot: Option<Snapshot>,
     ) -> Result<Self> {
         trace_scoped!("Vm::new_from_memory_manager");
@@ -597,12 +591,9 @@ impl Vm {
         device_manager
             .lock()
             .unwrap()
-            .create_devices(serial_pty, console_pty, console_resize_pipe)
+            .create_devices(serial_pty, console_pty, console_resize_pipe, on_tty)
             .map_err(Error::DeviceManager)?;
 
-        // SAFETY: trivially safe
-        let on_tty = unsafe { libc::isatty(libc::STDIN_FILENO) } != 0;
-
         #[cfg(feature = "tdx")]
         let kernel = config
             .lock()
@@ -644,7 +635,6 @@ impl Vm {
             initramfs,
             device_manager,
             config,
-            on_tty,
             threads: Vec::with_capacity(1),
             signals: None,
             state: RwLock::new(vm_state),
@@ -754,6 +744,7 @@ impl Vm {
         serial_pty: Option<PtyPair>,
         console_pty: Option<PtyPair>,
         console_resize_pipe: Option<File>,
+        on_tty: Arc<AtomicBool>,
         snapshot: Option<Snapshot>,
         source_url: Option<&str>,
         prefault: Option<bool>,
@@ -823,6 +814,7 @@ impl Vm {
             serial_pty,
             console_pty,
             console_resize_pipe,
+            on_tty,
             snapshot,
         )
     }
@@ -1213,15 +1205,6 @@ impl Vm {
 
         state.valid_transition(new_state)?;
 
-        if self.on_tty {
-            // Don't forget to set the terminal in canonical mode
-            // before to exit.
-            io::stdin()
-                .lock()
-                .set_canon_mode()
-                .map_err(Error::SetTerminalCanon)?;
-        }
-
         // Trigger the termination of the signal_handler thread
         if let Some(signals) = self.signals.take() {
             signals.close();
@@ -1983,17 +1966,6 @@ impl Vm {
         Ok(())
     }
 
-    fn setup_tty(&self) -> Result<()> {
-        if self.on_tty {
-            io::stdin()
-                .lock()
-                .set_raw_mode()
-                .map_err(Error::SetTerminalRaw)?;
-        }
-
-        Ok(())
-    }
-
     // Creates ACPI tables
     // In case of TDX being used, this is a no-op since the tables will be
     // created and passed when populating the HOB.
@@ -2048,7 +2020,6 @@ impl Vm {
         let rsdp_addr = self.create_acpi_tables();
 
         self.setup_signal_handler()?;
-        self.setup_tty()?;
 
         // Load kernel synchronously or if asynchronous then wait for load to
         // finish.
@@ -2163,7 +2134,6 @@ impl Vm {
             .map_err(Error::CpuManager)?;
 
         self.setup_signal_handler()?;
-        self.setup_tty()?;
 
         event!("vm", "restored");
         Ok(())

vmm/src/vm_config.rs

@@ -183,7 +183,7 @@ impl Default for MemoryConfig {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
 pub enum VhostMode {
     #[default]
     Client,
@@ -248,7 +248,7 @@ impl Default for DiskConfig {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct NetConfig {
     #[serde(default = "default_netconfig_tap")]
     pub tap: Option<String>,
@@ -563,7 +563,7 @@ pub struct TpmConfig {
     pub socket: PathBuf,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct VmConfig {
     #[serde(default)]
     pub cpus: CpusConfig,
@@ -596,4 +596,10 @@ pub struct VmConfig {
     pub gdb: bool,
     pub platform: Option<PlatformConfig>,
     pub tpm: Option<TpmConfig>,
+    // Preseved FDs are the ones that share the same life-time as its holding
+    // VmConfig instance, such as FDs for creating TAP devices.
+    // Perserved FDs will stay open as long as the holding VmConfig instance is
+    // valid, and will be closed when the holding VmConfig instance is destroyed.
+    #[serde(skip)]
+    pub preserved_fds: Option<Vec<i32>>,
 }