diff --git a/pci/src/vfio.rs b/pci/src/vfio.rs
index 85b3d75dc..cd468e8dc 100644
--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -1699,6 +1699,9 @@ impl VfioPciDevice {
                     error!("Could not remove the userspace memory region: {}", e);
                 }
 
+                self.memory_slot_allocator
+                    .free_memory_slot(user_memory_region.slot);
+
                 // SAFETY: FFI call with correct arguments
                 let ret = unsafe {
                     libc::munmap(
diff --git a/pci/src/vfio_user.rs b/pci/src/vfio_user.rs
index ce3e70f9e..1cd481421 100644
--- a/pci/src/vfio_user.rs
+++ b/pci/src/vfio_user.rs
@@ -222,6 +222,9 @@ impl VfioUserPciDevice {
                     error!("Could not remove the userspace memory region: {}", e);
                 }
 
+                self.memory_slot_allocator
+                    .free_memory_slot(user_memory_region.slot);
+
                 // Remove mmaps
                 // SAFETY: FFI call with correct arguments
                 let ret = unsafe {