devmapper: add pool device manager

Signed-off-by: Maksym Pavlenko <makpav@amazon.com>
This commit is contained in:
Maksym Pavlenko 2019-02-14 13:37:50 -08:00
parent 6e0ae68e17
commit 3a75882520
No known key found for this signature in database
GPG Key ID: BDA48CBFE7A0FC14
2 changed files with 552 additions and 0 deletions

View File

@ -0,0 +1,321 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package devmapper
import (
"context"
"os"
"path/filepath"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/snapshots/devmapper/dmsetup"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
)
// PoolDevice ties together data and metadata volumes, represents thin-pool and manages volumes, snapshots and device ids.
type PoolDevice struct {
poolName string
metadata *PoolMetadata
}
// NewPoolDevice creates new thin-pool from existing data and metadata volumes.
// If pool 'poolName' already exists, it'll be reloaded with new parameters.
func NewPoolDevice(ctx context.Context, config *Config) (*PoolDevice, error) {
log.G(ctx).Infof("initializing pool device %q", config.PoolName)
version, err := dmsetup.Version()
if err != nil {
log.G(ctx).Errorf("dmsetup not available")
return nil, err
}
log.G(ctx).Infof("using dmsetup:\n%s", version)
dbpath := filepath.Join(config.RootPath, config.PoolName+".db")
poolMetaStore, err := NewPoolMetadata(dbpath)
if err != nil {
return nil, err
}
if err := openPool(ctx, config); err != nil {
return nil, err
}
return &PoolDevice{
poolName: config.PoolName,
metadata: poolMetaStore,
}, nil
}
func openPool(ctx context.Context, config *Config) error {
if err := config.Validate(); err != nil {
return err
}
var (
poolPath = dmsetup.GetFullDevicePath(config.PoolName)
poolExists = false
)
if _, err := os.Stat(poolPath); err != nil && !os.IsNotExist(err) {
return errors.Wrapf(err, "failed to stat for %q", poolPath)
} else if err == nil {
poolExists = true
}
// Create new pool if not exists
if !poolExists {
log.G(ctx).Debug("creating new pool device")
if err := dmsetup.CreatePool(config.PoolName, config.DataDevice, config.MetadataDevice, config.DataBlockSizeSectors); err != nil {
return errors.Wrapf(err, "failed to create thin-pool with name %q", config.PoolName)
}
return nil
}
// Pool exists, check if it needs to be reloaded
if config.DataDevice != "" && config.MetadataDevice != "" {
log.G(ctx).Debugf("reloading existing pool %q", poolPath)
if err := dmsetup.ReloadPool(config.PoolName, config.DataDevice, config.MetadataDevice, config.DataBlockSizeSectors); err != nil {
return errors.Wrapf(err, "failed to reload pool %q", config.PoolName)
}
return nil
}
// If data and meta devices are not provided, use existing pool. Query info to make sure it's OK.
if _, err := dmsetup.Info(poolPath); err != nil {
return errors.Wrapf(err, "failed to query info for existing pool %q", poolPath)
}
return nil
}
// transition invokes 'updateStateFn' callback to perform devmapper operation and reflects device state changes/errors in meta store.
// 'tryingState' will be set before invoking callback. If callback succeeded 'successState' will be set, otherwise
// error details will be recorded in meta store.
func (p *PoolDevice) transition(ctx context.Context, deviceName string, tryingState DeviceState, successState DeviceState, updateStateFn func() error) error {
// Set device to trying state
uerr := p.metadata.UpdateDevice(ctx, deviceName, func(deviceInfo *DeviceInfo) error {
deviceInfo.State = tryingState
return nil
})
if uerr != nil {
return errors.Wrapf(uerr, "failed to set device %q state to %q", deviceName, tryingState)
}
var result *multierror.Error
// Invoke devmapper operation
err := updateStateFn()
if err != nil {
result = multierror.Append(result, err)
}
// If operation succeeded transition to success state, otherwise save error details
uerr = p.metadata.UpdateDevice(ctx, deviceName, func(deviceInfo *DeviceInfo) error {
if err == nil {
deviceInfo.State = successState
deviceInfo.Error = ""
} else {
deviceInfo.Error = err.Error()
}
return nil
})
if uerr != nil {
result = multierror.Append(result, uerr)
}
return result.ErrorOrNil()
}
// CreateThinDevice creates new devmapper thin-device with given name and size.
// Device ID for thin-device will be allocated from metadata store.
// If allocation successful, device will be activated with /dev/mapper/<deviceName>
func (p *PoolDevice) CreateThinDevice(ctx context.Context, deviceName string, virtualSizeBytes uint64) error {
info := &DeviceInfo{
Name: deviceName,
Size: virtualSizeBytes,
State: Unknown,
}
// Save initial device metadata and allocate new device ID from store
if err := p.metadata.AddDevice(ctx, info); err != nil {
return errors.Wrapf(err, "failed to save initial metadata for new thin device %q", deviceName)
}
// Create thin device
if err := p.transition(ctx, deviceName, Creating, Created, func() error {
return dmsetup.CreateDevice(p.poolName, info.DeviceID)
}); err != nil {
return errors.Wrapf(err, "failed to create new thin device %q (dev: %d)", info.Name, info.DeviceID)
}
// Activate thin device
if err := p.transition(ctx, deviceName, Activating, Activated, func() error {
return dmsetup.ActivateDevice(p.poolName, info.Name, info.DeviceID, info.Size, "")
}); err != nil {
return errors.Wrapf(err, "failed to activate new thin device %q (dev: %d)", info.Name, info.DeviceID)
}
return nil
}
// CreateSnapshotDevice creates and activates new thin-device from parent thin-device (makes snapshot)
func (p *PoolDevice) CreateSnapshotDevice(ctx context.Context, deviceName string, snapshotName string, virtualSizeBytes uint64) error {
baseInfo, err := p.metadata.GetDevice(ctx, deviceName)
if err != nil {
return errors.Wrapf(err, "failed to query device metadata for %q", deviceName)
}
isActivated := baseInfo.State == Activated
// Suspend thin device if it was activated previously
if isActivated {
if err := p.transition(ctx, baseInfo.Name, Suspending, Suspended, func() error {
return dmsetup.SuspendDevice(baseInfo.Name)
}); err != nil {
return errors.Wrapf(err, "failed to suspend device %q", baseInfo.Name)
}
}
snapInfo := &DeviceInfo{
Name: snapshotName,
Size: virtualSizeBytes,
ParentName: deviceName,
State: Unknown,
}
// Save snapshot metadata and allocate new device ID
if err := p.metadata.AddDevice(ctx, snapInfo); err != nil {
return errors.Wrapf(err, "failed to save initial metadata for snapshot %q", snapshotName)
}
// Create thin device snapshot
if err := p.transition(ctx, snapInfo.Name, Creating, Created, func() error {
return dmsetup.CreateSnapshot(p.poolName, snapInfo.DeviceID, baseInfo.DeviceID)
}); err != nil {
return errors.Wrapf(err,
"failed to create snapshot %q (dev: %d) from %q (dev: %d, activated: %t)",
snapInfo.Name,
snapInfo.DeviceID,
baseInfo.Name,
baseInfo.DeviceID,
isActivated)
}
if isActivated {
// Resume base thin-device
if err := p.transition(ctx, baseInfo.Name, Resuming, Resumed, func() error {
return dmsetup.ResumeDevice(baseInfo.Name)
}); err != nil {
return errors.Wrapf(err, "failed to resume device %q", deviceName)
}
}
// Activate snapshot
if err := p.transition(ctx, snapInfo.Name, Activating, Activated, func() error {
return dmsetup.ActivateDevice(p.poolName, snapInfo.Name, snapInfo.DeviceID, snapInfo.Size, "")
}); err != nil {
return errors.Wrapf(err, "failed to activate snapshot device %q (dev: %d)", snapInfo.Name, snapInfo.DeviceID)
}
return nil
}
// DeactivateDevice deactivates thin device
func (p *PoolDevice) DeactivateDevice(ctx context.Context, deviceName string, deferred bool) error {
devicePath := dmsetup.GetFullDevicePath(deviceName)
if _, err := os.Stat(devicePath); err != nil {
if os.IsNotExist(err) {
return ErrNotFound
}
return err
}
opts := []dmsetup.RemoveDeviceOpt{dmsetup.RemoveWithForce, dmsetup.RemoveWithRetries}
if deferred {
opts = append(opts, dmsetup.RemoveDeferred)
}
if err := p.transition(ctx, deviceName, Deactivating, Deactivated, func() error {
return dmsetup.RemoveDevice(deviceName, opts...)
}); err != nil {
return errors.Wrapf(err, "failed to deactivate device %q", deviceName)
}
return nil
}
// RemoveDevice completely wipes out thin device from thin-pool and frees it's device ID
func (p *PoolDevice) RemoveDevice(ctx context.Context, deviceName string) error {
info, err := p.metadata.GetDevice(ctx, deviceName)
if err != nil {
return errors.Wrapf(err, "can't query metadata for device %q", deviceName)
}
if err := p.DeactivateDevice(ctx, deviceName, true); err != nil && err != ErrNotFound {
return err
}
if err := p.transition(ctx, deviceName, Removing, Removed, func() error {
// Send 'delete' message to thin-pool
return dmsetup.DeleteDevice(p.poolName, info.DeviceID)
}); err != nil {
return errors.Wrapf(err, "failed to delete device %q (dev id: %d)", info.Name, info.DeviceID)
}
// Remove record from meta store and free device ID
if err := p.metadata.RemoveDevice(ctx, deviceName); err != nil {
return errors.Wrapf(err, "can't remove device %q metadata from store after removal", deviceName)
}
return nil
}
// RemovePool deactivates all child thin-devices and removes thin-pool device
func (p *PoolDevice) RemovePool(ctx context.Context) error {
deviceNames, err := p.metadata.GetDeviceNames(ctx)
if err != nil {
return errors.Wrap(err, "can't query device names")
}
var result *multierror.Error
// Deactivate devices if any
for _, name := range deviceNames {
if err := p.DeactivateDevice(ctx, name, true); err != nil && err != ErrNotFound {
result = multierror.Append(result, errors.Wrapf(err, "failed to remove %q", name))
}
}
if err := dmsetup.RemoveDevice(p.poolName, dmsetup.RemoveWithForce, dmsetup.RemoveWithRetries, dmsetup.RemoveDeferred); err != nil {
result = multierror.Append(result, errors.Wrapf(err, "failed to remove pool %q", p.poolName))
}
return result.ErrorOrNil()
}
// Close closes pool device (thin-pool will not be removed)
func (p *PoolDevice) Close() error {
return p.metadata.Close()
}

View File

@ -0,0 +1,231 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package devmapper
import (
"context"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"testing"
"github.com/containerd/containerd/pkg/testutil"
"github.com/containerd/containerd/snapshots/devmapper/dmsetup"
"github.com/containerd/containerd/snapshots/devmapper/losetup"
"github.com/docker/go-units"
"github.com/sirupsen/logrus"
"gotest.tools/assert"
)
const (
thinDevice1 = "thin-1"
thinDevice2 = "thin-2"
snapDevice1 = "snap-1"
device1Size = 100000
device2Size = 200000
testsPrefix = "devmapper-snapshotter-tests-"
)
// TestPoolDevice runs integration tests for pool device.
// The following scenario implemented:
// - Create pool device with name 'test-pool-device'
// - Create two thin volumes 'thin-1' and 'thin-2'
// - Write ext4 file system on 'thin-1' and make sure it'errs moutable
// - Write v1 test file on 'thin-1' volume
// - Take 'thin-1' snapshot 'snap-1'
// - Change v1 file to v2 on 'thin-1'
// - Mount 'snap-1' and make sure test file is v1
// - Unmount volumes and remove all devices
func TestPoolDevice(t *testing.T) {
testutil.RequiresRoot(t)
logrus.SetLevel(logrus.DebugLevel)
ctx := context.Background()
tempDir, err := ioutil.TempDir("", "pool-device-test-")
assert.NilError(t, err, "couldn't get temp directory for testing")
_, loopDataDevice := createLoopbackDevice(t, tempDir)
_, loopMetaDevice := createLoopbackDevice(t, tempDir)
defer func() {
// Detach loop devices and remove images
err := losetup.DetachLoopDevice(loopDataDevice, loopMetaDevice)
assert.NilError(t, err)
err = os.RemoveAll(tempDir)
assert.NilError(t, err, "couldn't cleanup temp directory")
}()
config := &Config{
PoolName: "test-pool-device-1",
RootPath: tempDir,
DataDevice: loopDataDevice,
MetadataDevice: loopMetaDevice,
DataBlockSize: "65536",
DataBlockSizeSectors: 128,
BaseImageSize: "16mb",
BaseImageSizeBytes: 16 * 1024 * 1024,
}
pool, err := NewPoolDevice(ctx, config)
assert.NilError(t, err, "can't create device pool")
assert.Assert(t, pool != nil)
defer func() {
err := pool.RemovePool(ctx)
assert.NilError(t, err, "can't close device pool")
}()
// Create thin devices
t.Run("CreateThinDevice", func(t *testing.T) {
testCreateThinDevice(t, pool)
})
// Make ext4 filesystem on 'thin-1'
t.Run("MakeFileSystem", func(t *testing.T) {
testMakeFileSystem(t, pool)
})
// Mount 'thin-1'
thin1MountPath := tempMountPath(t)
output, err := exec.Command("mount", dmsetup.GetFullDevicePath(thinDevice1), thin1MountPath).CombinedOutput()
assert.NilError(t, err, "failed to mount '%s': %s", thinDevice1, string(output))
// Write v1 test file on 'thin-1' device
thin1TestFilePath := filepath.Join(thin1MountPath, "TEST")
err = ioutil.WriteFile(thin1TestFilePath, []byte("test file (v1)"), 0700)
assert.NilError(t, err, "failed to write test file v1 on '%s' volume", thinDevice1)
// Take snapshot of 'thin-1'
t.Run("CreateSnapshotDevice", func(t *testing.T) {
testCreateSnapshot(t, pool)
})
// Update TEST file on 'thin-1' to v2
err = ioutil.WriteFile(thin1TestFilePath, []byte("test file (v2)"), 0700)
assert.NilError(t, err, "failed to write test file v2 on 'thin-1' volume after taking snapshot")
// Mount 'snap-1' and make sure TEST file is v1
snap1MountPath := tempMountPath(t)
output, err = exec.Command("mount", dmsetup.GetFullDevicePath(snapDevice1), snap1MountPath).CombinedOutput()
assert.NilError(t, err, "failed to mount '%s' device: %s", snapDevice1, string(output))
// Read test file from snapshot device and make sure it's v1
fileData, err := ioutil.ReadFile(filepath.Join(snap1MountPath, "TEST"))
assert.NilError(t, err, "couldn't read test file from '%s' device", snapDevice1)
assert.Assert(t, string(fileData) == "test file (v1)", "test file content is invalid on snapshot")
// Unmount devices before removing
output, err = exec.Command("umount", thin1MountPath, snap1MountPath).CombinedOutput()
assert.NilError(t, err, "failed to unmount devices: %s", string(output))
t.Run("DeactivateDevice", func(t *testing.T) {
testDeactivateThinDevice(t, pool)
})
t.Run("RemoveDevice", func(t *testing.T) {
testRemoveThinDevice(t, pool)
})
}
func testCreateThinDevice(t *testing.T, pool *PoolDevice) {
ctx := context.Background()
err := pool.CreateThinDevice(ctx, thinDevice1, device1Size)
assert.NilError(t, err, "can't create first thin device")
err = pool.CreateThinDevice(ctx, thinDevice1, device1Size)
assert.Assert(t, err != nil, "device pool allows duplicated device names")
err = pool.CreateThinDevice(ctx, thinDevice2, device2Size)
assert.NilError(t, err, "can't create second thin device")
deviceInfo1, err := pool.metadata.GetDevice(ctx, thinDevice1)
assert.NilError(t, err)
deviceInfo2, err := pool.metadata.GetDevice(ctx, thinDevice2)
assert.NilError(t, err)
assert.Assert(t, deviceInfo1.DeviceID != deviceInfo2.DeviceID, "assigned device ids should be different")
}
func testMakeFileSystem(t *testing.T, pool *PoolDevice) {
devicePath := dmsetup.GetFullDevicePath(thinDevice1)
args := []string{
devicePath,
"-E",
"nodiscard,lazy_itable_init=0,lazy_journal_init=0",
}
output, err := exec.Command("mkfs.ext4", args...).CombinedOutput()
assert.NilError(t, err, "failed to make filesystem on '%s': %s", thinDevice1, string(output))
}
func testCreateSnapshot(t *testing.T, pool *PoolDevice) {
err := pool.CreateSnapshotDevice(context.Background(), thinDevice1, snapDevice1, device1Size)
assert.NilError(t, err, "failed to create snapshot from '%s' volume", thinDevice1)
}
func testDeactivateThinDevice(t *testing.T, pool *PoolDevice) {
deviceList := []string{
thinDevice2,
snapDevice1,
}
for _, deviceName := range deviceList {
err := pool.DeactivateDevice(context.Background(), deviceName, false)
assert.NilError(t, err, "failed to remove '%s'", deviceName)
}
err := pool.DeactivateDevice(context.Background(), "not-existing-device", false)
assert.Assert(t, err != nil, "should return an error if trying to remove not existing device")
}
func testRemoveThinDevice(t *testing.T, pool *PoolDevice) {
err := pool.RemoveDevice(testCtx, thinDevice1)
assert.NilError(t, err, "should delete thin device from pool")
}
func tempMountPath(t *testing.T) string {
path, err := ioutil.TempDir("", "devmapper-snapshotter-mount-")
assert.NilError(t, err, "failed to get temp directory for mount")
return path
}
func createLoopbackDevice(t *testing.T, dir string) (string, string) {
file, err := ioutil.TempFile(dir, testsPrefix)
assert.NilError(t, err)
size, err := units.RAMInBytes("128Mb")
assert.NilError(t, err)
err = file.Truncate(size)
assert.NilError(t, err)
err = file.Close()
assert.NilError(t, err)
imagePath := file.Name()
loopDevice, err := losetup.AttachLoopDevice(imagePath)
assert.NilError(t, err)
return imagePath, loopDevice
}