devmapper: async remove device using Cleanup

Fix: #3923
Signed-off-by: Eric Ren <renzhen@linux.alibaba.com>

snapshots/devmapper/snapshotter.go

@@ -303,9 +303,18 @@ func (s *Snapshotter) removeDevice(ctx context.Context, key string) error {
 	}
 
 	deviceName := s.getDeviceName(snapID)
-	if err := s.pool.RemoveDevice(ctx, deviceName); err != nil {
-		log.G(ctx).WithError(err).Errorf("failed to remove device")
-		return err
+	if !s.config.AsyncRemove {
+		if err := s.pool.RemoveDevice(ctx, deviceName); err != nil {
+			log.G(ctx).WithError(err).Errorf("failed to remove device")
+			return err
+		}
+	} else {
+		// The asynchronous cleanup will do the real device remove work.
+		log.G(ctx).WithField("device", deviceName).Debug("async remove")
+		if err := s.pool.MarkDeviceState(ctx, deviceName, Removed); err != nil {
+			log.G(ctx).WithError(err).Errorf("failed to mark device as removed")
+			return err
+		}
 	}
 
 	return nil
@@ -486,3 +495,34 @@ func (s *Snapshotter) withTransaction(ctx context.Context, writable bool, fn fun
 
 	return nil
 }
+
+func (s *Snapshotter) Cleanup(ctx context.Context) error {
+	var removedDevices []*DeviceInfo
+
+	if !s.config.AsyncRemove {
+		return nil
+	}
+
+	if err := s.pool.WalkDevices(ctx, func(info *DeviceInfo) error {
+		if info.State == Removed {
+			removedDevices = append(removedDevices, info)
+		}
+		return nil
+	}); err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to query devices from metastore")
+		return err
+	}
+
+	var result *multierror.Error
+	for _, dev := range removedDevices {
+		log.G(ctx).WithField("device", dev.Name).Debug("cleanup device")
+		if err := s.pool.RemoveDevice(ctx, dev.Name); err != nil {
+			log.G(ctx).WithField("device", dev.Name).Error("failed to cleanup device")
+			result = multierror.Append(result, err)
+		} else {
+			log.G(ctx).WithField("device", dev.Name).Debug("cleanuped device")
+		}
+	}
+
+	return result.ErrorOrNil()
+}