vmm: config: Validate mergeable and shared are not both set

KSM will not attempt to merge pages that are mapped as MAP_SHARED, so
configuring memory with both mergeable and shared options is invalid.
Add validation to reject configurations where both options are enabled
for memory or memory zones.

Signed-off-by: Rob Bradford <rbradford@meta.com>

vmm/src/config.rs

@@ -404,6 +404,9 @@ pub enum ValidationError {
     /// The supplied PCI ID is reserved
     #[error("Given PCI device ID ({0}) is reserved")]
     ReservedPciDeviceId(u8),
+    /// Invalid to set both 'mergeable' and 'shared' for memory
+    #[error("Invalid to set both 'mergeable' and 'shared' for memory")]
+    InvalidSharedMemoryWithMergeable,
 }
 
 type ValidationResult<T> = std::result::Result<T, ValidationError>;
@@ -3016,6 +3019,18 @@ impl VmConfig {
             }
         }
 
+        if self.memory.shared && self.memory.mergeable {
+            return Err(ValidationError::InvalidSharedMemoryWithMergeable);
+        }
+
+        if let Some(zones) = &self.memory.zones {
+            for zone in zones {
+                if zone.shared && zone.mergeable {
+                    return Err(ValidationError::InvalidSharedMemoryWithMergeable);
+                }
+            }
+        }
+
         if let Some(user_devices) = &self.user_devices {
             if !user_devices.is_empty() && !self.backed_by_shared_memory() {
                 return Err(ValidationError::UserDevicesRequireSharedMemory);
@@ -5225,6 +5240,29 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             Err(ValidationError::InvalidHugePageSize(3 << 20))
         );
 
+        // Test mergeable and shared validation for global memory
+        let mut invalid_config = valid_config.clone();
+        invalid_config.memory.shared = true;
+        invalid_config.memory.mergeable = true;
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::InvalidSharedMemoryWithMergeable)
+        );
+
+        // Test mergeable and shared validation for memory zones
+        let mut invalid_config = valid_config.clone();
+        invalid_config.memory.zones = Some(vec![MemoryZoneConfig {
+            id: "mem0".to_string(),
+            size: 1 << 30,
+            shared: true,
+            mergeable: true,
+            ..Default::default()
+        }]);
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::InvalidSharedMemoryWithMergeable)
+        );
+
         let mut still_valid_config = valid_config.clone();
         still_valid_config.platform = Some(platform_fixture());
         still_valid_config.validate().unwrap();