build: Release v36.1 (bug fix release)

Signed-off-by: Bo Chen <chen.bo@intel.com>

Cargo.lock

@@ -427,7 +427,7 @@ checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
 
 [[package]]
 name = "cloud-hypervisor"
-version = "36.0.0"
+version = "36.1.0"
 dependencies = [
  "anyhow",
  "api_client",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "36.0.0"
+version = "36.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"

arch/src/x86_64/mod.rs

@@ -1311,7 +1311,7 @@ fn update_cpuid_topology(
 // sections exposed to the guest.
 fn update_cpuid_sgx(
     cpuid: &mut Vec<CpuIdEntry>,
-    epc_sections: &Vec<SgxEpcSection>,
+    epc_sections: &[SgxEpcSection],
 ) -> Result<(), Error> {
     // Something's wrong if there's no EPC section.
     if epc_sections.is_empty() {

block/src/raw_async_aio.rs

@@ -120,15 +120,20 @@ impl AsyncIo for RawFileAsyncAio {
     }
 
     fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        let iocbs = [&mut aio::IoControlBlock {
-            aio_fildes: self.fd.as_raw_fd() as u32,
-            aio_lio_opcode: aio::IOCB_CMD_FSYNC as u16,
-            aio_data: user_data.unwrap_or(0),
-            aio_flags: aio::IOCB_FLAG_RESFD,
-            aio_resfd: self.eventfd.as_raw_fd() as u32,
-            ..Default::default()
-        }];
-        let _ = self.ctx.submit(&iocbs[..]).map_err(AsyncIoError::Fsync)?;
+        if let Some(user_data) = user_data {
+            let iocbs = [&mut aio::IoControlBlock {
+                aio_fildes: self.fd.as_raw_fd() as u32,
+                aio_lio_opcode: aio::IOCB_CMD_FSYNC as u16,
+                aio_data: user_data,
+                aio_flags: aio::IOCB_FLAG_RESFD,
+                aio_resfd: self.eventfd.as_raw_fd() as u32,
+                ..Default::default()
+            }];
+            let _ = self.ctx.submit(&iocbs[..]).map_err(AsyncIoError::Fsync)?;
+        } else {
+            // SAFETY: FFI call with a valid fd
+            unsafe { libc::fsync(self.fd) };
+        }
 
         Ok(())
     }

devices/src/pvpanic.rs

@@ -26,6 +26,7 @@ const PVPANIC_VENDOR_ID: u16 = 0x1b36;
 const PVPANIC_DEVICE_ID: u16 = 0x0011;
 
 pub const PVPANIC_DEVICE_MMIO_SIZE: u64 = 0x2;
+pub const PVPANIC_DEVICE_MMIO_ALIGNMENT: u64 = 0x10;
 
 const PVPANIC_PANICKED: u8 = 1 << 0;
 const PVPANIC_CRASH_LOADED: u8 = 1 << 1;
@@ -192,7 +193,7 @@ impl PciDevice for PvPanicDevice {
         let bar_addr = allocator
             .lock()
             .unwrap()
-            .allocate_mmio_hole_addresses(None, region_size, None)
+            .allocate_mmio_hole_addresses(None, region_size, Some(PVPANIC_DEVICE_MMIO_ALIGNMENT))
             .ok_or(PciDeviceError::IoAllocationFailed(region_size))?;
 
         let bar = PciBarConfiguration::default()

release-notes.md

@@ -1,3 +1,4 @@
+- [v36.1](#v361)
 - [v36.0](#v360)
     - [Command Line Changes](#command-line-changes)
     - [Enabled Features Reported via API Endpoint and CLI](#enabled-features-reported-via-api-endpoint-and-cli)
@@ -310,6 +311,16 @@
     - [Unit testing](#unit-testing)
     - [Integration tests parallelization](#integration-tests-parallelization)
 
+# v36.1
+
+This is a bug fix release. The following issues have been addressed:
+
+* Fix aio backend behavior for block devices when writeback cache
+disabled (#5930)
+* Fix PvPanic device PCI BAR alignment (#5956)
+* Bug fix to OpenAPI specification file (#5967)
+* Error out early for live migration when TDX is enabled (#6025)
+
 # v36.0
 
 This release has been tracked in our [roadmap

resources/Dockerfile

@@ -3,7 +3,7 @@
 # When changing this file don't forget to update the tag name in the
 # .github/workflows/docker-image.yaml file if doing multiple per day
 
-FROM ubuntu:20.04 as dev
+FROM ubuntu:22.04 as dev
 
 ARG TARGETARCH
 ARG RUST_TOOLCHAIN="1.67.1"
@@ -43,7 +43,6 @@ RUN apt-get update \
         socat \
         dosfstools \
         cpio \
-        python \
         python3 \
         python3-setuptools \
         ntfs-3g \

scripts/dev_cli.sh

@@ -7,7 +7,7 @@
 CLI_NAME="Cloud Hypervisor"
 
 CTR_IMAGE_TAG="ghcr.io/cloud-hypervisor/cloud-hypervisor"
-CTR_IMAGE_VERSION="20231012-0"
+CTR_IMAGE_VERSION="20231220-0"
 : "${CTR_IMAGE:=${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}}"
 
 DOCKER_RUNTIME="docker"
@@ -285,8 +285,7 @@ cmd_build() {
     rustflags="$RUSTFLAGS"
     target_cc=""
     if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
-        rustflags="$rustflags -C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-        target_cc="musl-gcc"
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
     fi
 
     $DOCKER_RUNTIME run \
@@ -399,8 +398,7 @@ cmd_tests() {
     rustflags="$RUSTFLAGS"
     target_cc=""
     if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
-        rustflags="$rustflags -C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-        target_cc="musl-gcc"
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
     fi
 
     if [[ "$unit" = true ]]; then

scripts/run_integration_tests_aarch64.sh

@@ -20,7 +20,7 @@ build_spdk_nvme() {
         sed -i "/grpcio/d" scripts/pkgdep/debian.sh
         ./scripts/pkgdep.sh
         ./configure --with-vfio-user
-        chmod +x /usr/local/lib/python3.8/dist-packages/ninja/data/bin/ninja
+        chmod +x /usr/local/lib/python3.10/dist-packages/ninja/data/bin/ninja
         make -j `nproc` || exit 1
         touch .built
         popd
@@ -30,7 +30,7 @@ build_spdk_nvme() {
     fi
     cp "$WORKLOADS_DIR/spdk/build/bin/nvmf_tgt" $SPDK_DEPLOY_DIR/nvmf_tgt
     cp "$WORKLOADS_DIR/spdk/scripts/rpc.py" $SPDK_DEPLOY_DIR/rpc.py
-    cp -r "$WORKLOADS_DIR/spdk/scripts/rpc" $SPDK_DEPLOY_DIR/rpc
+    cp -r "$WORKLOADS_DIR/spdk/python/spdk/" $SPDK_DEPLOY_DIR/
     cp -r "$WORKLOADS_DIR/spdk/python" $SPDK_DEPLOY_DIR/../
 }
 
@@ -38,7 +38,7 @@ build_virtiofsd() {
     VIRTIOFSD_DIR="$WORKLOADS_DIR/virtiofsd_build"
     VIRTIOFSD_REPO="https://gitlab.com/virtio-fs/virtiofsd.git"
 
-    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.1.0 "220405d7a2606c92636d31992b5cb3036a41047b"
+    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.8.0 "97ea7908fe7f9bc59916671a771bdcfaf4044b45"
 
     if [ ! -f "$VIRTIOFSD_DIR/.built" ]; then
         pushd $VIRTIOFSD_DIR
@@ -159,12 +159,25 @@ update_workloads() {
     popd
 
     # Download Cloud Hypervisor binary from its last stable release
-    LAST_RELEASE_VERSION="v34.0"
+    LAST_RELEASE_VERSION="v36.0"
     CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64"
     CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
     pushd $WORKLOADS_DIR
-    time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
-    chmod +x $CH_RELEASE_NAME
+    # Repeat a few times to workaround a random wget failure
+    WGET_RETRY_MAX=10
+    wget_retry=0
+
+    until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]
+    do
+        time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break
+        wget_retry=$((wget_retry+1))
+    done
+
+    if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then
+        exit 1
+    else
+        chmod +x $CH_RELEASE_NAME
+    fi
     popd
 
     # Build custom kernel for guest VMs

scripts/run_integration_tests_live_migration.sh

@@ -44,7 +44,7 @@ fi
 popd
 
 # Download Cloud Hypervisor binary from its last stable release
-LAST_RELEASE_VERSION="v34.0"
+LAST_RELEASE_VERSION="v36.0"
 CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static"
 CH_RELEASE_NAME="cloud-hypervisor-static"
 pushd $WORKLOADS_DIR

scripts/run_integration_tests_x86_64.sh

@@ -122,7 +122,7 @@ if [ ! -f "$VIRTIOFSD" ]; then
     pushd $WORKLOADS_DIR
     git clone "https://gitlab.com/virtio-fs/virtiofsd.git" $VIRTIOFSD_DIR
     pushd $VIRTIOFSD_DIR
-    git checkout v1.1.0
+    git checkout v1.8.0
     time cargo build --release
     cp target/release/virtiofsd $VIRTIOFSD || exit 1
     popd

test_infra/src/lib.rs

@@ -90,7 +90,7 @@ impl GuestNetworkConfig {
             None => DEFAULT_TCP_LISTENER_TIMEOUT,
         };
 
-        match (|| -> Result<(), WaitForBootError> {
+        let mut closure = || -> Result<(), WaitForBootError> {
             let listener =
                 TcpListener::bind(listen_addr.as_str()).map_err(WaitForBootError::Listen)?;
             listener
@@ -143,7 +143,9 @@ impl GuestNetworkConfig {
                     Err(WaitForBootError::Accept(e))
                 }
             }
-        })() {
+        };
+
+        match closure() {
             Err(e) => {
                 let duration = start.elapsed();
                 eprintln!(
@@ -559,7 +561,7 @@ fn scp_to_guest_with_auth(
 ) -> Result<(), SshCommandError> {
     let mut counter = 0;
     loop {
-        match (|| -> Result<(), SshCommandError> {
+        let closure = || -> Result<(), SshCommandError> {
             let tcp =
                 TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?;
             let mut sess = Session::new().unwrap();
@@ -592,7 +594,9 @@ fn scp_to_guest_with_auth(
             let _ = channel.wait_close();
 
             Ok(())
-        })() {
+        };
+
+        match closure() {
             Ok(_) => break,
             Err(e) => {
                 counter += 1;
@@ -647,7 +651,7 @@ pub fn ssh_command_ip_with_auth(
 
     let mut counter = 0;
     loop {
-        match (|| -> Result<(), SshCommandError> {
+        let mut closure = || -> Result<(), SshCommandError> {
             let tcp =
                 TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?;
             let mut sess = Session::new().unwrap();
@@ -676,7 +680,9 @@ pub fn ssh_command_ip_with_auth(
             } else {
                 Ok(())
             }
-        })() {
+        };
+
+        match closure() {
             Ok(_) => break,
             Err(e) => {
                 counter += 1;

tests/integration.rs

@@ -3405,16 +3405,19 @@ mod common_parallel {
     }
 
     #[test]
+    #[cfg(not(target_arch = "aarch64"))]
     fn test_vhost_user_blk_default() {
         test_vhost_user_blk(2, false, false, Some(&prepare_vubd))
     }
 
     #[test]
+    #[cfg(not(target_arch = "aarch64"))]
     fn test_vhost_user_blk_readonly() {
         test_vhost_user_blk(1, true, false, Some(&prepare_vubd))
     }
 
     #[test]
+    #[cfg(not(target_arch = "aarch64"))]
     fn test_vhost_user_blk_direct() {
         test_vhost_user_blk(1, false, true, Some(&prepare_vubd))
     }
@@ -3929,7 +3932,7 @@ mod common_parallel {
         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
         let guest = Guest::new(Box::new(focal));
 
-        let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output");
+        let serial_path = guest.tmp_dir.as_path().join("serial-output");
         #[cfg(target_arch = "x86_64")]
         let console_str: &str = "console=ttyS0";
         #[cfg(target_arch = "aarch64")]
@@ -4042,8 +4045,8 @@ mod common_parallel {
     fn test_serial_socket_interaction() {
         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
         let guest = Guest::new(Box::new(focal));
-        let serial_socket = guest.tmp_dir.as_path().join("/tmp/serial.socket");
-        let serial_socket_pty = guest.tmp_dir.as_path().join("/tmp/serial.pty");
+        let serial_socket = guest.tmp_dir.as_path().join("serial.socket");
+        let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty");
         let serial_option = if cfg!(target_arch = "x86_64") {
             " console=ttyS0"
         } else {
@@ -4151,7 +4154,7 @@ mod common_parallel {
         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
         let guest = Guest::new(Box::new(focal));
 
-        let console_path = guest.tmp_dir.as_path().join("/tmp/console-output");
+        let console_path = guest.tmp_dir.as_path().join("console-output");
         let mut child = GuestCommand::new(&guest)
             .args(["--cpus", "boot=1"])
             .args(["--memory", "size=512M"])
@@ -4212,8 +4215,8 @@ mod common_parallel {
     fn test_vfio() {
         setup_vfio_network_interfaces();
 
-        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
-        let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0);
+        let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string());
+        let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0);
 
         let mut workload_path = dirs::home_dir().unwrap();
         workload_path.push("workloads");
@@ -7929,6 +7932,7 @@ mod windows {
     }
 
     #[test]
+    #[ignore = "See #6037"]
     #[cfg(not(feature = "mshv"))]
     #[cfg(not(target_arch = "aarch64"))]
     fn test_windows_guest_disk_hotplug() {
@@ -8024,6 +8028,7 @@ mod windows {
     }
 
     #[test]
+    #[ignore = "See #6037"]
     #[cfg(not(feature = "mshv"))]
     #[cfg(not(target_arch = "aarch64"))]
     fn test_windows_guest_disk_hotplug_multi() {
@@ -9458,51 +9463,43 @@ mod live_migration {
         }
 
         #[test]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_basic() {
             _test_live_migration(true, false)
         }
 
         #[test]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_local() {
             _test_live_migration(true, true)
         }
 
         #[test]
         #[cfg(not(feature = "mshv"))]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_numa() {
             _test_live_migration_numa(true, false)
         }
 
         #[test]
         #[cfg(not(feature = "mshv"))]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_numa_local() {
             _test_live_migration_numa(true, true)
         }
 
         #[test]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_watchdog() {
             _test_live_migration_watchdog(true, false)
         }
 
         #[test]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_watchdog_local() {
             _test_live_migration_watchdog(true, true)
         }
 
         #[test]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_balloon() {
             _test_live_migration_balloon(true, false)
         }
 
         #[test]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_balloon_local() {
             _test_live_migration_balloon(true, true)
         }
@@ -9515,6 +9512,7 @@ mod live_migration {
 
         // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup
         #[test]
+        #[ignore = "See #5532"]
         #[cfg(target_arch = "x86_64")]
         #[cfg(not(feature = "mshv"))]
         fn test_live_migration_ovs_dpdk() {
@@ -9529,9 +9527,9 @@ mod live_migration {
         }
 
         #[test]
+        #[ignore = "See #5532"]
         #[cfg(target_arch = "x86_64")]
         #[cfg(not(feature = "mshv"))]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_ovs_dpdk() {
             _test_live_migration_ovs_dpdk(true, false);
         }
@@ -9539,7 +9537,6 @@ mod live_migration {
         #[test]
         #[cfg(target_arch = "x86_64")]
         #[cfg(not(feature = "mshv"))]
-        #[ignore = "See #5791"]
         fn test_live_upgrade_ovs_dpdk_local() {
             _test_live_migration_ovs_dpdk(true, true);
         }

virtio-devices/src/vhost_user/vu_common_ctrl.rs

@@ -7,7 +7,6 @@ use crate::{
     get_host_address_range, GuestMemoryMmap, GuestRegionMmap, MmapRegion, VirtioInterrupt,
     VirtioInterruptType,
 };
-use std::convert::TryInto;
 use std::ffi;
 use std::fs::File;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
@@ -201,7 +200,7 @@ impl VhostUserHandle {
 
         let mut vrings_info = Vec::new();
         for (queue_index, queue, queue_evt) in queues.iter() {
-            let actual_size: usize = queue.size().try_into().unwrap();
+            let actual_size: usize = queue.size().into();
 
             let config_data = VringConfigData {
                 queue_max_size: queue.max_size(),

vm-migration/src/lib.rs

@@ -13,7 +13,7 @@ pub mod protocol;
 
 /// Global VMM version for versioning
 const MAJOR_VERSION: u16 = 36;
-const MINOR_VERSION: u16 = 0;
+const MINOR_VERSION: u16 = 1;
 const VMM_VERSION: u16 = MAJOR_VERSION << 12 | MINOR_VERSION & 0b1111;
 
 pub trait VersionMapped {

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -347,6 +347,28 @@ paths:
         500:
           description: The new vDPA device could not be added to the VM instance.
 
+  /vm.add-user-device:
+    put:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/VmAddUserDevice'
+        description: The path of the new device
+        required: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PciDeviceInfo'
+          description: The new device was successfully added to the VM instance.
+        "204":
+          description: The new device was successfully (cold) added to the VM instance.
+        "404":
+          description: The new device could not be added to the VM instance.
+      summary: Add a new userspace device to the VM
+
   /vm.snapshot:
     put:
       summary: Returns a VM snapshot.
@@ -1156,3 +1178,11 @@ components:
           type: string
         local:
           type: boolean
+
+    VmAddUserDevice:
+      required:
+        - socket
+      type: object
+      properties:
+        socket:
+          type: string

vmm/src/lib.rs

@@ -1664,7 +1664,12 @@ impl Vmm {
         #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
         let common_cpuid = {
             #[cfg(feature = "tdx")]
-            let tdx = vm_config.lock().unwrap().is_tdx_enabled();
+            if vm_config.lock().unwrap().is_tdx_enabled() {
+                return Err(MigratableError::MigrateSend(anyhow!(
+                    "Live Migration is not supported when TDX is enabled"
+                )));
+            };
+
             let amx = vm_config.lock().unwrap().cpus.features.amx;
             let phys_bits =
                 vm::physical_bits(&hypervisor, vm_config.lock().unwrap().cpus.max_phys_bits);
@@ -1675,7 +1680,7 @@ impl Vmm {
                     phys_bits,
                     kvm_hyperv: vm_config.lock().unwrap().cpus.kvm_hyperv,
                     #[cfg(feature = "tdx")]
-                    tdx,
+                    tdx: false,
                     amx,
                 },
             )
@@ -1854,6 +1859,13 @@ impl Vmm {
         src_vm_config: &Arc<Mutex<VmConfig>>,
         src_vm_cpuid: &[hypervisor::arch::x86::CpuIdEntry],
     ) -> result::Result<(), MigratableError> {
+        #[cfg(feature = "tdx")]
+        if src_vm_config.lock().unwrap().is_tdx_enabled() {
+            return Err(MigratableError::MigrateReceive(anyhow!(
+                "Live Migration is not supported when TDX is enabled"
+            )));
+        };
+
         // We check the `CPUID` compatibility of between the source vm and destination, which is
         // mostly about feature compatibility and "topology/sgx" leaves are not relevant.
         let dest_cpuid = &{
@@ -1867,7 +1879,7 @@ impl Vmm {
                     phys_bits,
                     kvm_hyperv: vm_config.cpus.kvm_hyperv,
                     #[cfg(feature = "tdx")]
-                    tdx: vm_config.is_tdx_enabled(),
+                    tdx: false,
                     amx: vm_config.cpus.features.amx,
                 },
             )

vmm/src/vm.rs

@@ -2390,12 +2390,9 @@ impl Snapshottable for Vm {
     fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
         event!("vm", "snapshotting");
 
-        #[cfg(feature = "tdx")]
-        let tdx_enabled = self.config.lock().unwrap().is_tdx_enabled();
-
         #[cfg(feature = "tdx")]
         {
-            if tdx_enabled {
+            if self.config.lock().unwrap().is_tdx_enabled() {
                 return Err(MigratableError::Snapshot(anyhow!(
                     "Snapshot not possible with TDX VM"
                 )));
@@ -2423,7 +2420,7 @@ impl Snapshottable for Vm {
                     phys_bits,
                     kvm_hyperv: self.config.lock().unwrap().cpus.kvm_hyperv,
                     #[cfg(feature = "tdx")]
-                    tdx: tdx_enabled,
+                    tdx: false,
                     amx,
                 },
             )