misc: Use a more relaxed memory model when possible

When a total ordering between multiple atomic variables is not required
then use Ordering::Acquire with atomic loads and Ordering::Release with
atomic stores.

This will improve performance as this does not require a memory fence
on x86_64 which Ordering::SeqCst will use.

Add a comment to the code in the vCPU handling code where it operates on
multiple atomics to explain why Ordering::SeqCst is required.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

virtio-devices/src/transport/pci_common_config.rs

@@ -64,7 +64,7 @@ impl VirtioPciCommonConfig {
             device_feature_select: self.device_feature_select,
             driver_feature_select: self.driver_feature_select,
             queue_select: self.queue_select,
-            msix_config: self.msix_config.load(Ordering::SeqCst),
+            msix_config: self.msix_config.load(Ordering::Acquire),
         }
     }
 
@@ -74,7 +74,7 @@ impl VirtioPciCommonConfig {
         self.device_feature_select = state.device_feature_select;
         self.driver_feature_select = state.driver_feature_select;
         self.queue_select = state.queue_select;
-        self.msix_config.store(state.msix_config, Ordering::SeqCst);
+        self.msix_config.store(state.msix_config, Ordering::Release);
     }
 
     pub fn read(
@@ -153,7 +153,7 @@ impl VirtioPciCommonConfig {
     fn read_common_config_word(&self, offset: u64, queues: &[Queue]) -> u16 {
         debug!("read_common_config_word: offset 0x{:x}", offset);
         match offset {
-            0x10 => self.msix_config.load(Ordering::SeqCst),
+            0x10 => self.msix_config.load(Ordering::Acquire),
             0x12 => queues.len() as u16, // num_queues
             0x16 => self.queue_select,
             0x18 => self.with_queue(queues, |q| q.size).unwrap_or(0),
@@ -176,7 +176,7 @@ impl VirtioPciCommonConfig {
     fn write_common_config_word(&mut self, offset: u64, value: u16, queues: &mut Vec<Queue>) {
         debug!("write_common_config_word: offset 0x{:x}", offset);
         match offset {
-            0x10 => self.msix_config.store(value, Ordering::SeqCst),
+            0x10 => self.msix_config.store(value, Ordering::Release),
             0x16 => self.queue_select = value,
             0x18 => self.with_queue_mut(queues, |q| q.size = value),
             0x1a => self.with_queue_mut(queues, |q| q.vector = value),

virtio-devices/src/transport/pci_device.rs

@@ -448,7 +448,7 @@ impl VirtioPciDevice {
     fn state(&self) -> VirtioPciDeviceState {
         VirtioPciDeviceState {
             device_activated: self.device_activated,
-            interrupt_status: self.interrupt_status.load(Ordering::SeqCst),
+            interrupt_status: self.interrupt_status.load(Ordering::Acquire),
             queues: self.queues.clone(),
         }
     }
@@ -456,7 +456,7 @@ impl VirtioPciDevice {
     fn set_state(&mut self, state: &VirtioPciDeviceState) -> std::result::Result<(), Error> {
         self.device_activated = state.device_activated;
         self.interrupt_status
-            .store(state.interrupt_status, Ordering::SeqCst);
+            .store(state.interrupt_status, Ordering::Release);
 
         // Update virtqueues indexes for both available and used rings.
         if let Some(mem) = self.memory.as_ref() {
@@ -685,7 +685,7 @@ impl VirtioInterrupt for VirtioInterruptMsix {
         queue: Option<&Queue>,
     ) -> std::result::Result<(), std::io::Error> {
         let vector = match int_type {
-            VirtioInterruptType::Config => self.config_vector.load(Ordering::SeqCst),
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire),
             VirtioInterruptType::Queue => {
                 if let Some(q) = queue {
                     q.vector
@@ -717,7 +717,7 @@ impl VirtioInterrupt for VirtioInterruptMsix {
 
     fn notifier(&self, int_type: &VirtioInterruptType, queue: Option<&Queue>) -> Option<&EventFd> {
         let vector = match int_type {
-            VirtioInterruptType::Config => self.config_vector.load(Ordering::SeqCst),
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire),
             VirtioInterruptType::Queue => {
                 if let Some(q) = queue {
                     q.vector
@@ -891,7 +891,7 @@ impl PciDevice for VirtioPciDevice {
             o if ISR_CONFIG_BAR_OFFSET <= o && o < ISR_CONFIG_BAR_OFFSET + ISR_CONFIG_SIZE => {
                 if let Some(v) = data.get_mut(0) {
                     // Reading this register resets it to 0.
-                    *v = self.interrupt_status.swap(0, Ordering::SeqCst) as u8;
+                    *v = self.interrupt_status.swap(0, Ordering::AcqRel) as u8;
                 }
             }
             o if DEVICE_CONFIG_BAR_OFFSET <= o
@@ -936,7 +936,7 @@ impl PciDevice for VirtioPciDevice {
             o if ISR_CONFIG_BAR_OFFSET <= o && o < ISR_CONFIG_BAR_OFFSET + ISR_CONFIG_SIZE => {
                 if let Some(v) = data.get(0) {
                     self.interrupt_status
-                        .fetch_and(!(*v as usize), Ordering::SeqCst);
+                        .fetch_and(!(*v as usize), Ordering::AcqRel);
                 }
             }
             o if DEVICE_CONFIG_BAR_OFFSET <= o