From 3a758825209a7e049db1617882bb35e780211238 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 14 Feb 2019 13:37:50 -0800 Subject: [PATCH] devmapper: add pool device manager Signed-off-by: Maksym Pavlenko --- snapshots/devmapper/pool_device.go | 321 ++++++++++++++++++++++++ snapshots/devmapper/pool_device_test.go | 231 +++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 snapshots/devmapper/pool_device.go create mode 100644 snapshots/devmapper/pool_device_test.go diff --git a/snapshots/devmapper/pool_device.go b/snapshots/devmapper/pool_device.go new file mode 100644 index 000000000..ceb48ab57 --- /dev/null +++ b/snapshots/devmapper/pool_device.go @@ -0,0 +1,321 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package devmapper + +import ( + "context" + "os" + "path/filepath" + + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/snapshots/devmapper/dmsetup" + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" +) + +// PoolDevice ties together data and metadata volumes, represents thin-pool and manages volumes, snapshots and device ids. +type PoolDevice struct { + poolName string + metadata *PoolMetadata +} + +// NewPoolDevice creates new thin-pool from existing data and metadata volumes. +// If pool 'poolName' already exists, it'll be reloaded with new parameters. +func NewPoolDevice(ctx context.Context, config *Config) (*PoolDevice, error) { + log.G(ctx).Infof("initializing pool device %q", config.PoolName) + + version, err := dmsetup.Version() + if err != nil { + log.G(ctx).Errorf("dmsetup not available") + return nil, err + } + + log.G(ctx).Infof("using dmsetup:\n%s", version) + + dbpath := filepath.Join(config.RootPath, config.PoolName+".db") + poolMetaStore, err := NewPoolMetadata(dbpath) + if err != nil { + return nil, err + } + + if err := openPool(ctx, config); err != nil { + return nil, err + } + + return &PoolDevice{ + poolName: config.PoolName, + metadata: poolMetaStore, + }, nil +} + +func openPool(ctx context.Context, config *Config) error { + if err := config.Validate(); err != nil { + return err + } + + var ( + poolPath = dmsetup.GetFullDevicePath(config.PoolName) + poolExists = false + ) + + if _, err := os.Stat(poolPath); err != nil && !os.IsNotExist(err) { + return errors.Wrapf(err, "failed to stat for %q", poolPath) + } else if err == nil { + poolExists = true + } + + // Create new pool if not exists + if !poolExists { + log.G(ctx).Debug("creating new pool device") + if err := dmsetup.CreatePool(config.PoolName, config.DataDevice, config.MetadataDevice, config.DataBlockSizeSectors); err != nil { + return errors.Wrapf(err, "failed to create thin-pool with name %q", config.PoolName) + } + + return nil + } + + // Pool exists, check if it needs to be reloaded + if config.DataDevice != "" && config.MetadataDevice != "" { + log.G(ctx).Debugf("reloading existing pool %q", poolPath) + if err := dmsetup.ReloadPool(config.PoolName, config.DataDevice, config.MetadataDevice, config.DataBlockSizeSectors); err != nil { + return errors.Wrapf(err, "failed to reload pool %q", config.PoolName) + } + + return nil + } + + // If data and meta devices are not provided, use existing pool. Query info to make sure it's OK. + if _, err := dmsetup.Info(poolPath); err != nil { + return errors.Wrapf(err, "failed to query info for existing pool %q", poolPath) + } + + return nil +} + +// transition invokes 'updateStateFn' callback to perform devmapper operation and reflects device state changes/errors in meta store. +// 'tryingState' will be set before invoking callback. If callback succeeded 'successState' will be set, otherwise +// error details will be recorded in meta store. +func (p *PoolDevice) transition(ctx context.Context, deviceName string, tryingState DeviceState, successState DeviceState, updateStateFn func() error) error { + // Set device to trying state + uerr := p.metadata.UpdateDevice(ctx, deviceName, func(deviceInfo *DeviceInfo) error { + deviceInfo.State = tryingState + return nil + }) + + if uerr != nil { + return errors.Wrapf(uerr, "failed to set device %q state to %q", deviceName, tryingState) + } + + var result *multierror.Error + + // Invoke devmapper operation + err := updateStateFn() + + if err != nil { + result = multierror.Append(result, err) + } + + // If operation succeeded transition to success state, otherwise save error details + uerr = p.metadata.UpdateDevice(ctx, deviceName, func(deviceInfo *DeviceInfo) error { + if err == nil { + deviceInfo.State = successState + deviceInfo.Error = "" + } else { + deviceInfo.Error = err.Error() + } + return nil + }) + + if uerr != nil { + result = multierror.Append(result, uerr) + } + + return result.ErrorOrNil() +} + +// CreateThinDevice creates new devmapper thin-device with given name and size. +// Device ID for thin-device will be allocated from metadata store. +// If allocation successful, device will be activated with /dev/mapper/ +func (p *PoolDevice) CreateThinDevice(ctx context.Context, deviceName string, virtualSizeBytes uint64) error { + info := &DeviceInfo{ + Name: deviceName, + Size: virtualSizeBytes, + State: Unknown, + } + + // Save initial device metadata and allocate new device ID from store + if err := p.metadata.AddDevice(ctx, info); err != nil { + return errors.Wrapf(err, "failed to save initial metadata for new thin device %q", deviceName) + } + + // Create thin device + if err := p.transition(ctx, deviceName, Creating, Created, func() error { + return dmsetup.CreateDevice(p.poolName, info.DeviceID) + }); err != nil { + return errors.Wrapf(err, "failed to create new thin device %q (dev: %d)", info.Name, info.DeviceID) + } + + // Activate thin device + if err := p.transition(ctx, deviceName, Activating, Activated, func() error { + return dmsetup.ActivateDevice(p.poolName, info.Name, info.DeviceID, info.Size, "") + }); err != nil { + return errors.Wrapf(err, "failed to activate new thin device %q (dev: %d)", info.Name, info.DeviceID) + } + + return nil +} + +// CreateSnapshotDevice creates and activates new thin-device from parent thin-device (makes snapshot) +func (p *PoolDevice) CreateSnapshotDevice(ctx context.Context, deviceName string, snapshotName string, virtualSizeBytes uint64) error { + baseInfo, err := p.metadata.GetDevice(ctx, deviceName) + if err != nil { + return errors.Wrapf(err, "failed to query device metadata for %q", deviceName) + } + + isActivated := baseInfo.State == Activated + + // Suspend thin device if it was activated previously + if isActivated { + if err := p.transition(ctx, baseInfo.Name, Suspending, Suspended, func() error { + return dmsetup.SuspendDevice(baseInfo.Name) + }); err != nil { + return errors.Wrapf(err, "failed to suspend device %q", baseInfo.Name) + } + } + + snapInfo := &DeviceInfo{ + Name: snapshotName, + Size: virtualSizeBytes, + ParentName: deviceName, + State: Unknown, + } + + // Save snapshot metadata and allocate new device ID + if err := p.metadata.AddDevice(ctx, snapInfo); err != nil { + return errors.Wrapf(err, "failed to save initial metadata for snapshot %q", snapshotName) + } + + // Create thin device snapshot + if err := p.transition(ctx, snapInfo.Name, Creating, Created, func() error { + return dmsetup.CreateSnapshot(p.poolName, snapInfo.DeviceID, baseInfo.DeviceID) + }); err != nil { + return errors.Wrapf(err, + "failed to create snapshot %q (dev: %d) from %q (dev: %d, activated: %t)", + snapInfo.Name, + snapInfo.DeviceID, + baseInfo.Name, + baseInfo.DeviceID, + isActivated) + } + + if isActivated { + // Resume base thin-device + if err := p.transition(ctx, baseInfo.Name, Resuming, Resumed, func() error { + return dmsetup.ResumeDevice(baseInfo.Name) + }); err != nil { + return errors.Wrapf(err, "failed to resume device %q", deviceName) + } + } + + // Activate snapshot + if err := p.transition(ctx, snapInfo.Name, Activating, Activated, func() error { + return dmsetup.ActivateDevice(p.poolName, snapInfo.Name, snapInfo.DeviceID, snapInfo.Size, "") + }); err != nil { + return errors.Wrapf(err, "failed to activate snapshot device %q (dev: %d)", snapInfo.Name, snapInfo.DeviceID) + } + + return nil +} + +// DeactivateDevice deactivates thin device +func (p *PoolDevice) DeactivateDevice(ctx context.Context, deviceName string, deferred bool) error { + devicePath := dmsetup.GetFullDevicePath(deviceName) + if _, err := os.Stat(devicePath); err != nil { + if os.IsNotExist(err) { + return ErrNotFound + } + + return err + } + + opts := []dmsetup.RemoveDeviceOpt{dmsetup.RemoveWithForce, dmsetup.RemoveWithRetries} + if deferred { + opts = append(opts, dmsetup.RemoveDeferred) + } + + if err := p.transition(ctx, deviceName, Deactivating, Deactivated, func() error { + return dmsetup.RemoveDevice(deviceName, opts...) + }); err != nil { + return errors.Wrapf(err, "failed to deactivate device %q", deviceName) + } + + return nil +} + +// RemoveDevice completely wipes out thin device from thin-pool and frees it's device ID +func (p *PoolDevice) RemoveDevice(ctx context.Context, deviceName string) error { + info, err := p.metadata.GetDevice(ctx, deviceName) + if err != nil { + return errors.Wrapf(err, "can't query metadata for device %q", deviceName) + } + + if err := p.DeactivateDevice(ctx, deviceName, true); err != nil && err != ErrNotFound { + return err + } + + if err := p.transition(ctx, deviceName, Removing, Removed, func() error { + // Send 'delete' message to thin-pool + return dmsetup.DeleteDevice(p.poolName, info.DeviceID) + }); err != nil { + return errors.Wrapf(err, "failed to delete device %q (dev id: %d)", info.Name, info.DeviceID) + } + + // Remove record from meta store and free device ID + if err := p.metadata.RemoveDevice(ctx, deviceName); err != nil { + return errors.Wrapf(err, "can't remove device %q metadata from store after removal", deviceName) + } + + return nil +} + +// RemovePool deactivates all child thin-devices and removes thin-pool device +func (p *PoolDevice) RemovePool(ctx context.Context) error { + deviceNames, err := p.metadata.GetDeviceNames(ctx) + if err != nil { + return errors.Wrap(err, "can't query device names") + } + + var result *multierror.Error + + // Deactivate devices if any + for _, name := range deviceNames { + if err := p.DeactivateDevice(ctx, name, true); err != nil && err != ErrNotFound { + result = multierror.Append(result, errors.Wrapf(err, "failed to remove %q", name)) + } + } + + if err := dmsetup.RemoveDevice(p.poolName, dmsetup.RemoveWithForce, dmsetup.RemoveWithRetries, dmsetup.RemoveDeferred); err != nil { + result = multierror.Append(result, errors.Wrapf(err, "failed to remove pool %q", p.poolName)) + } + + return result.ErrorOrNil() +} + +// Close closes pool device (thin-pool will not be removed) +func (p *PoolDevice) Close() error { + return p.metadata.Close() +} diff --git a/snapshots/devmapper/pool_device_test.go b/snapshots/devmapper/pool_device_test.go new file mode 100644 index 000000000..d18dbffe0 --- /dev/null +++ b/snapshots/devmapper/pool_device_test.go @@ -0,0 +1,231 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package devmapper + +import ( + "context" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/containerd/containerd/pkg/testutil" + "github.com/containerd/containerd/snapshots/devmapper/dmsetup" + "github.com/containerd/containerd/snapshots/devmapper/losetup" + "github.com/docker/go-units" + "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +const ( + thinDevice1 = "thin-1" + thinDevice2 = "thin-2" + snapDevice1 = "snap-1" + device1Size = 100000 + device2Size = 200000 + testsPrefix = "devmapper-snapshotter-tests-" +) + +// TestPoolDevice runs integration tests for pool device. +// The following scenario implemented: +// - Create pool device with name 'test-pool-device' +// - Create two thin volumes 'thin-1' and 'thin-2' +// - Write ext4 file system on 'thin-1' and make sure it'errs moutable +// - Write v1 test file on 'thin-1' volume +// - Take 'thin-1' snapshot 'snap-1' +// - Change v1 file to v2 on 'thin-1' +// - Mount 'snap-1' and make sure test file is v1 +// - Unmount volumes and remove all devices +func TestPoolDevice(t *testing.T) { + testutil.RequiresRoot(t) + + logrus.SetLevel(logrus.DebugLevel) + ctx := context.Background() + + tempDir, err := ioutil.TempDir("", "pool-device-test-") + assert.NilError(t, err, "couldn't get temp directory for testing") + + _, loopDataDevice := createLoopbackDevice(t, tempDir) + _, loopMetaDevice := createLoopbackDevice(t, tempDir) + + defer func() { + // Detach loop devices and remove images + err := losetup.DetachLoopDevice(loopDataDevice, loopMetaDevice) + assert.NilError(t, err) + + err = os.RemoveAll(tempDir) + assert.NilError(t, err, "couldn't cleanup temp directory") + }() + + config := &Config{ + PoolName: "test-pool-device-1", + RootPath: tempDir, + DataDevice: loopDataDevice, + MetadataDevice: loopMetaDevice, + DataBlockSize: "65536", + DataBlockSizeSectors: 128, + BaseImageSize: "16mb", + BaseImageSizeBytes: 16 * 1024 * 1024, + } + + pool, err := NewPoolDevice(ctx, config) + assert.NilError(t, err, "can't create device pool") + assert.Assert(t, pool != nil) + + defer func() { + err := pool.RemovePool(ctx) + assert.NilError(t, err, "can't close device pool") + }() + + // Create thin devices + t.Run("CreateThinDevice", func(t *testing.T) { + testCreateThinDevice(t, pool) + }) + + // Make ext4 filesystem on 'thin-1' + t.Run("MakeFileSystem", func(t *testing.T) { + testMakeFileSystem(t, pool) + }) + + // Mount 'thin-1' + thin1MountPath := tempMountPath(t) + output, err := exec.Command("mount", dmsetup.GetFullDevicePath(thinDevice1), thin1MountPath).CombinedOutput() + assert.NilError(t, err, "failed to mount '%s': %s", thinDevice1, string(output)) + + // Write v1 test file on 'thin-1' device + thin1TestFilePath := filepath.Join(thin1MountPath, "TEST") + err = ioutil.WriteFile(thin1TestFilePath, []byte("test file (v1)"), 0700) + assert.NilError(t, err, "failed to write test file v1 on '%s' volume", thinDevice1) + + // Take snapshot of 'thin-1' + t.Run("CreateSnapshotDevice", func(t *testing.T) { + testCreateSnapshot(t, pool) + }) + + // Update TEST file on 'thin-1' to v2 + err = ioutil.WriteFile(thin1TestFilePath, []byte("test file (v2)"), 0700) + assert.NilError(t, err, "failed to write test file v2 on 'thin-1' volume after taking snapshot") + + // Mount 'snap-1' and make sure TEST file is v1 + snap1MountPath := tempMountPath(t) + output, err = exec.Command("mount", dmsetup.GetFullDevicePath(snapDevice1), snap1MountPath).CombinedOutput() + assert.NilError(t, err, "failed to mount '%s' device: %s", snapDevice1, string(output)) + + // Read test file from snapshot device and make sure it's v1 + fileData, err := ioutil.ReadFile(filepath.Join(snap1MountPath, "TEST")) + assert.NilError(t, err, "couldn't read test file from '%s' device", snapDevice1) + assert.Assert(t, string(fileData) == "test file (v1)", "test file content is invalid on snapshot") + + // Unmount devices before removing + output, err = exec.Command("umount", thin1MountPath, snap1MountPath).CombinedOutput() + assert.NilError(t, err, "failed to unmount devices: %s", string(output)) + + t.Run("DeactivateDevice", func(t *testing.T) { + testDeactivateThinDevice(t, pool) + }) + + t.Run("RemoveDevice", func(t *testing.T) { + testRemoveThinDevice(t, pool) + }) +} + +func testCreateThinDevice(t *testing.T, pool *PoolDevice) { + ctx := context.Background() + + err := pool.CreateThinDevice(ctx, thinDevice1, device1Size) + assert.NilError(t, err, "can't create first thin device") + + err = pool.CreateThinDevice(ctx, thinDevice1, device1Size) + assert.Assert(t, err != nil, "device pool allows duplicated device names") + + err = pool.CreateThinDevice(ctx, thinDevice2, device2Size) + assert.NilError(t, err, "can't create second thin device") + + deviceInfo1, err := pool.metadata.GetDevice(ctx, thinDevice1) + assert.NilError(t, err) + + deviceInfo2, err := pool.metadata.GetDevice(ctx, thinDevice2) + assert.NilError(t, err) + + assert.Assert(t, deviceInfo1.DeviceID != deviceInfo2.DeviceID, "assigned device ids should be different") +} + +func testMakeFileSystem(t *testing.T, pool *PoolDevice) { + devicePath := dmsetup.GetFullDevicePath(thinDevice1) + args := []string{ + devicePath, + "-E", + "nodiscard,lazy_itable_init=0,lazy_journal_init=0", + } + + output, err := exec.Command("mkfs.ext4", args...).CombinedOutput() + assert.NilError(t, err, "failed to make filesystem on '%s': %s", thinDevice1, string(output)) +} + +func testCreateSnapshot(t *testing.T, pool *PoolDevice) { + err := pool.CreateSnapshotDevice(context.Background(), thinDevice1, snapDevice1, device1Size) + assert.NilError(t, err, "failed to create snapshot from '%s' volume", thinDevice1) +} + +func testDeactivateThinDevice(t *testing.T, pool *PoolDevice) { + deviceList := []string{ + thinDevice2, + snapDevice1, + } + + for _, deviceName := range deviceList { + err := pool.DeactivateDevice(context.Background(), deviceName, false) + assert.NilError(t, err, "failed to remove '%s'", deviceName) + } + + err := pool.DeactivateDevice(context.Background(), "not-existing-device", false) + assert.Assert(t, err != nil, "should return an error if trying to remove not existing device") +} + +func testRemoveThinDevice(t *testing.T, pool *PoolDevice) { + err := pool.RemoveDevice(testCtx, thinDevice1) + assert.NilError(t, err, "should delete thin device from pool") +} + +func tempMountPath(t *testing.T) string { + path, err := ioutil.TempDir("", "devmapper-snapshotter-mount-") + assert.NilError(t, err, "failed to get temp directory for mount") + + return path +} + +func createLoopbackDevice(t *testing.T, dir string) (string, string) { + file, err := ioutil.TempFile(dir, testsPrefix) + assert.NilError(t, err) + + size, err := units.RAMInBytes("128Mb") + assert.NilError(t, err) + + err = file.Truncate(size) + assert.NilError(t, err) + + err = file.Close() + assert.NilError(t, err) + + imagePath := file.Name() + + loopDevice, err := losetup.AttachLoopDevice(imagePath) + assert.NilError(t, err) + + return imagePath, loopDevice +}