diff --git a/core/mount/mount_linux.go b/core/mount/mount_linux.go
index 330760335..ec5417ed7 100644
--- a/core/mount/mount_linux.go
+++ b/core/mount/mount_linux.go
@@ -251,7 +251,9 @@ func doPrepareIDMappedOverlay(lowerDirs []string, usernsFd int) (tmpLowerDirs []
 	}
 	cleanUp := func() {
 		for _, lowerDir := range tmpLowerDirs {
-			if err := unix.Unmount(lowerDir, 0); err != nil {
+			// Do a detached unmount so even if the resource is busy, the mount will be
+			// gone (eventually) and we can safely delete the directory too.
+			if err := unix.Unmount(lowerDir, unix.MNT_DETACH); err != nil {
 				log.L.WithError(err).Warnf("failed to unmount temp lowerdir %s", lowerDir)
 			}
 		}