From 970a5a410d6015cdefe9eb018f0e514f704592ed Mon Sep 17 00:00:00 2001
From: Henry Wang <Henry.Wang@arm.com>
Date: Sun, 6 Sep 2020 10:23:39 +0800
Subject: [PATCH] vmm: decouple vCPU init from `configure_vcpus`

Since calling `KVM_GET_ONE_REG` before `KVM_VCPU_INIT` will
result in an error: Exec format error (os error 8). This commit
decouples the vCPU init process from `configure_vcpus`. Therefore
in the process of restoring the vCPUs, these vCPUs can be
initialized separately before started.

Signed-off-by: Henry Wang <Henry.Wang@arm.com>
---
 arch/src/aarch64/mod.rs | 21 ---------------------
 vmm/src/cpu.rs          | 33 ++++++++++++++++++++++++++++++---
 2 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs
index 7a7019ac0..850b363e9 100644
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -15,7 +15,6 @@ pub use self::fdt::DeviceInfoForFDT;
 use crate::DeviceType;
 use crate::RegionType;
 use aarch64::gic::GICDevice;
-use hypervisor::kvm::kvm_bindings;
 use std::collections::HashMap;
 use std::ffi::CStr;
 use std::fmt::Debug;
@@ -42,12 +41,6 @@ pub enum Error {
 
     /// Error configuring the MPIDR register
     VcpuRegMPIDR(hypervisor::HypervisorCpuError),
-
-    /// Error fetching prefered target
-    VcpuArmPreferredTarget(hypervisor::HypervisorVmError),
-
-    /// Error doing Vcpu Init on Arm.
-    VcpuArmInit(hypervisor::HypervisorCpuError),
 }
 
 impl From<Error> for super::Error {
@@ -68,23 +61,9 @@ pub struct EntryPoint {
 pub fn configure_vcpu(
     fd: &Arc<dyn hypervisor::Vcpu>,
     id: u8,
-    vm: &Arc<dyn hypervisor::Vm>,
     kernel_entry_point: Option<EntryPoint>,
     vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
 ) -> super::Result<u64> {
-    let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
-
-    // This reads back the kernel's preferred target type.
-    vm.get_preferred_target(&mut kvi)
-        .map_err(Error::VcpuArmPreferredTarget)?;
-    // We already checked that the capability is supported.
-    kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PSCI_0_2;
-    // Non-boot cpus are powered off initially.
-    if id > 0 {
-        kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
-    }
-
-    fd.vcpu_init(&kvi).map_err(Error::VcpuArmInit)?;
     if let Some(kernel_entry_point) = kernel_entry_point {
         regs::setup_regs(
             fd,
diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs
index 05e522fe0..98370998a 100644
--- a/vmm/src/cpu.rs
+++ b/vmm/src/cpu.rs
@@ -29,6 +29,8 @@ use arch::EntryPoint;
 #[cfg(target_arch = "x86_64")]
 use arch::{CpuidPatch, CpuidReg};
 use devices::interrupt_controller::InterruptController;
+#[cfg(target_arch = "aarch64")]
+use hypervisor::kvm::kvm_bindings;
 #[cfg(target_arch = "x86_64")]
 use hypervisor::CpuId;
 use hypervisor::{CpuState, VmExit};
@@ -127,6 +129,14 @@ pub enum Error {
     /// Error configuring VCPU
     VcpuConfiguration(arch::Error),
 
+    #[cfg(target_arch = "aarch64")]
+    /// Error fetching prefered target
+    VcpuArmPreferredTarget(hypervisor::HypervisorVmError),
+
+    #[cfg(target_arch = "aarch64")]
+    /// Error doing vCPU init on Arm.
+    VcpuArmInit(hypervisor::HypervisorCpuError),
+
     /// Failed to join on vCPU threads
     ThreadCleanup(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
 
@@ -303,9 +313,9 @@ impl Vcpu {
     ) -> Result<()> {
         #[cfg(target_arch = "aarch64")]
         {
-            self.mpidr =
-                arch::configure_vcpu(&self.vcpu, self.id, vm, kernel_entry_point, vm_memory)
-                    .map_err(Error::VcpuConfiguration)?;
+            self.init(vm)?;
+            self.mpidr = arch::configure_vcpu(&self.vcpu, self.id, kernel_entry_point, vm_memory)
+                .map_err(Error::VcpuConfiguration)?;
         }
 
         #[cfg(target_arch = "x86_64")]
@@ -334,6 +344,23 @@ impl Vcpu {
         self.saved_state.clone()
     }
 
+    /// Initializes an aarch64 specific vcpu for booting Linux.
+    #[cfg(target_arch = "aarch64")]
+    pub fn init(&self, vm: &Arc<dyn hypervisor::Vm>) -> Result<()> {
+        let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
+
+        // This reads back the kernel's preferred target type.
+        vm.get_preferred_target(&mut kvi)
+            .map_err(Error::VcpuArmPreferredTarget)?;
+        // We already checked that the capability is supported.
+        kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PSCI_0_2;
+        // Non-boot cpus are powered off initially.
+        if self.id > 0 {
+            kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
+        }
+        self.vcpu.vcpu_init(&kvi).map_err(Error::VcpuArmInit)
+    }
+
     /// Runs the VCPU until it exits, returning the reason.
     ///
     /// Note that the state of the VCPU and associated VM must be setup first for this to do