github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add retries when deleting a devmapper device

Browse Source 
Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>
This commit is contained in:
Maksym Pavlenko 2020-12-09 09:13:34 -08:00
parent 1e624fa3de
commit 2b87d4554f
1 changed files with 41 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
snapshots/devmapper/pool_device.go
View File

@ -75,6 +75,33 @@ func NewPoolDevice(ctx context.Context, config *Config) (*PoolDevice, error) {
return poolDevice, nil return poolDevice, nil
} }
func retry(ctx context.Context, f func() error) error {
var (
maxRetries = 100
retryDelay = 100 * time.Millisecond
retryErr error
)
for attempt := 1; attempt <= maxRetries; attempt++ {
retryErr = f()
if retryErr == nil {
return nil
} else if retryErr != unix.EBUSY {
return retryErr
}
// Don't spam logs
if attempt%10 == 0 {
log.G(ctx).WithError(retryErr).Warnf("retrying... (%d of %d)", attempt, maxRetries)
}
// Devmapper device is busy, give it a bit of time and retry removal
time.Sleep(retryDelay)
}
return retryErr
}
// ensureDeviceStates updates devices to their real state: // ensureDeviceStates updates devices to their real state:
// - marks devices with incomplete states (after crash) as 'Faulty' // - marks devices with incomplete states (after crash) as 'Faulty'
// - activates devices if they are marked as 'Activated' but the dm // - activates devices if they are marked as 'Activated' but the dm
@ -393,30 +420,13 @@ func (p *PoolDevice) DeactivateDevice(ctx context.Context, deviceName string, de
} }
if err := p.transition(ctx, deviceName, Deactivating, Deactivated, func() error { if err := p.transition(ctx, deviceName, Deactivating, Deactivated, func() error {
var ( return retry(ctx, func() error {
maxRetries = 100 if err := dmsetup.RemoveDevice(deviceName, opts...); err != nil {
retryDelay = 100 * time.Millisecond return errors.Wrap(err, "failed to deactivate device")
retryErr error }
)
for attempt := 1; attempt <= maxRetries; attempt++ {
retryErr = dmsetup.RemoveDevice(deviceName, opts...)
if retryErr == nil {
return nil return nil
} else if retryErr != unix.EBUSY { })
return retryErr
}
// Don't spam logs
if attempt%10 == 0 {
log.G(ctx).WithError(retryErr).Warnf("failed to deactivate device, retrying... (%d of %d)", attempt, maxRetries)
}
// Devmapper device is busy, give it a bit of time and retry removal
time.Sleep(retryDelay)
}
return retryErr
}); err != nil { }); err != nil {
return errors.Wrapf(err, "failed to deactivate device %q", deviceName) return errors.Wrapf(err, "failed to deactivate device %q", deviceName)
} }
@ -493,14 +503,15 @@ func (p *PoolDevice) RemoveDevice(ctx context.Context, deviceName string) error
func (p *PoolDevice) deleteDevice(ctx context.Context, info *DeviceInfo) error { func (p *PoolDevice) deleteDevice(ctx context.Context, info *DeviceInfo) error {
if err := p.transition(ctx, info.Name, Removing, Removed, func() error { if err := p.transition(ctx, info.Name, Removing, Removed, func() error {
return retry(ctx, func() error {
// Send 'delete' message to thin-pool // Send 'delete' message to thin-pool
e := dmsetup.DeleteDevice(p.poolName, info.DeviceID) e := dmsetup.DeleteDevice(p.poolName, info.DeviceID)
// Ignores the error if the device has been deleted already. // Ignores the error if the device has been deleted already.
if e != nil && !errors.Is(e, unix.ENODATA) { if e != nil && !errors.Is(e, unix.ENODATA) {
return e return e
} }
return nil return nil
})
}); err != nil { }); err != nil {
return errors.Wrapf(err, "failed to delete device %q (dev id: %d)", info.Name, info.DeviceID) return errors.Wrapf(err, "failed to delete device %q (dev id: %d)", info.Name, info.DeviceID)
} }