devmapper: add dmsetup

Signed-off-by: Maksym Pavlenko <makpav@amazon.com>

snapshots/devmapper/dmsetup/dmsetup.go

@@ -0,0 +1,341 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package dmsetup
+
+import (
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// DevMapperDir represents devmapper devices location
+	DevMapperDir = "/dev/mapper/"
+	// SectorSize represents the number of bytes in one sector on devmapper devices
+	SectorSize = 512
+)
+
+// DeviceInfo represents device info returned by "dmsetup info".
+// dmsetup(8) provides more information on each of these fields.
+type DeviceInfo struct {
+	Name            string
+	BlockDeviceName string
+	TableLive       bool
+	TableInactive   bool
+	Suspended       bool
+	ReadOnly        bool
+	Major           uint32
+	Minor           uint32
+	OpenCount       uint32 // Open reference count
+	TargetCount     uint32 // Number of targets in the live table
+	EventNumber     uint32 // Last event sequence number (used by wait)
+}
+
+var errTable map[string]unix.Errno
+
+func init() {
+	// Precompute map of <text>=<errno> for optimal lookup
+	errTable = make(map[string]unix.Errno)
+	for errno := unix.EPERM; errno <= unix.EHWPOISON; errno++ {
+		errTable[errno.Error()] = errno
+	}
+}
+
+// CreatePool creates a device with the given name, data and metadata file and block size (see "dmsetup create")
+func CreatePool(poolName, dataFile, metaFile string, blockSizeSectors uint32) error {
+	thinPool, err := makeThinPoolMapping(dataFile, metaFile, blockSizeSectors)
+	if err != nil {
+		return err
+	}
+
+	_, err = dmsetup("create", poolName, "--table", thinPool)
+	return err
+}
+
+// ReloadPool reloads existing thin-pool (see "dmsetup reload")
+func ReloadPool(deviceName, dataFile, metaFile string, blockSizeSectors uint32) error {
+	thinPool, err := makeThinPoolMapping(dataFile, metaFile, blockSizeSectors)
+	if err != nil {
+		return err
+	}
+
+	_, err = dmsetup("reload", deviceName, "--table", thinPool)
+	return err
+}
+
+const (
+	lowWaterMark = 32768                // Picked arbitrary, might need tuning
+	skipZeroing  = "skip_block_zeroing" // Skipping zeroing to reduce latency for device creation
+)
+
+// makeThinPoolMapping makes thin-pool table entry
+func makeThinPoolMapping(dataFile, metaFile string, blockSizeSectors uint32) (string, error) {
+	dataDeviceSizeBytes, err := BlockDeviceSize(dataFile)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to get block device size: %s", dataFile)
+	}
+
+	// Thin-pool mapping target has the following format:
+	// start - starting block in virtual device
+	// length - length of this segment
+	// metadata_dev - the metadata device
+	// data_dev - the data device
+	// data_block_size - the data block size in sectors
+	// low_water_mark - the low water mark, expressed in blocks of size data_block_size
+	// feature_args - the number of feature arguments
+	// args
+	lengthSectors := dataDeviceSizeBytes / SectorSize
+	target := fmt.Sprintf("0 %d thin-pool %s %s %d %d 1 %s",
+		lengthSectors,
+		metaFile,
+		dataFile,
+		blockSizeSectors,
+		lowWaterMark,
+		skipZeroing)
+
+	return target, nil
+}
+
+// CreateDevice sends "create_thin <deviceID>" message to the given thin-pool
+func CreateDevice(poolName string, deviceID uint32) error {
+	_, err := dmsetup("message", poolName, "0", fmt.Sprintf("create_thin %d", deviceID))
+	return err
+}
+
+// ActivateDevice activates the given thin-device using the 'thin' target
+func ActivateDevice(poolName string, deviceName string, deviceID uint32, size uint64, external string) error {
+	mapping := makeThinMapping(poolName, deviceID, size, external)
+	_, err := dmsetup("create", deviceName, "--table", mapping)
+	return err
+}
+
+// makeThinMapping makes thin target table entry
+func makeThinMapping(poolName string, deviceID uint32, sizeBytes uint64, externalOriginDevice string) string {
+	lengthSectors := sizeBytes / SectorSize
+
+	// Thin target has the following format:
+	// start - starting block in virtual device
+	// length - length of this segment
+	// pool_dev - the thin-pool device, can be /dev/mapper/pool_name or 253:0
+	// dev_id - the internal device id of the device to be activated
+	// external_origin_dev - an optional block device outside the pool to be treated as a read-only snapshot origin.
+	target := fmt.Sprintf("0 %d thin %s %d %s", lengthSectors, GetFullDevicePath(poolName), deviceID, externalOriginDevice)
+	return strings.TrimSpace(target)
+}
+
+// SuspendDevice suspends the given device (see "dmsetup suspend")
+func SuspendDevice(deviceName string) error {
+	_, err := dmsetup("suspend", deviceName)
+	return err
+}
+
+// ResumeDevice resumes the given device (see "dmsetup resume")
+func ResumeDevice(deviceName string) error {
+	_, err := dmsetup("resume", deviceName)
+	return err
+}
+
+// Table returns the current table for the device
+func Table(deviceName string) (string, error) {
+	return dmsetup("table", deviceName)
+}
+
+// CreateSnapshot sends "create_snap" message to the given thin-pool.
+// Caller needs to suspend and resume device if it is active.
+func CreateSnapshot(poolName string, deviceID uint32, baseDeviceID uint32) error {
+	_, err := dmsetup("message", poolName, "0", fmt.Sprintf("create_snap %d %d", deviceID, baseDeviceID))
+	return err
+}
+
+// DeleteDevice sends "delete <deviceID>" message to the given thin-pool
+func DeleteDevice(poolName string, deviceID uint32) error {
+	_, err := dmsetup("message", poolName, "0", fmt.Sprintf("delete %d", deviceID))
+	return err
+}
+
+// RemoveDeviceOpt represents command line arguments for "dmsetup remove" command
+type RemoveDeviceOpt string
+
+const (
+	// RemoveWithForce flag replaces the table with one that fails all I/O if
+	// open device can't be removed
+	RemoveWithForce RemoveDeviceOpt = "--force"
+	// RemoveWithRetries option will cause the operation to be retried
+	// for a few seconds before failing
+	RemoveWithRetries RemoveDeviceOpt = "--retry"
+	// RemoveDeferred flag will enable deferred removal of open devices,
+	// the device will be removed when the last user closes it
+	RemoveDeferred RemoveDeviceOpt = "--deferred"
+)
+
+// RemoveDevice removes a device (see "dmsetup remove")
+func RemoveDevice(deviceName string, opts ...RemoveDeviceOpt) error {
+	args := []string{
+		"remove",
+	}
+
+	for _, opt := range opts {
+		args = append(args, string(opt))
+	}
+
+	args = append(args, GetFullDevicePath(deviceName))
+
+	_, err := dmsetup(args...)
+	return err
+}
+
+// Info outputs device information (see "dmsetup info").
+// If device name is empty, all device infos will be returned.
+func Info(deviceName string) ([]*DeviceInfo, error) {
+	output, err := dmsetup(
+		"info",
+		"--columns",
+		"--noheadings",
+		"-o",
+		"name,blkdevname,attr,major,minor,open,segments,events",
+		"--separator",
+		" ",
+		deviceName)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		lines   = strings.Split(output, "\n")
+		devices = make([]*DeviceInfo, len(lines))
+	)
+
+	for i, line := range lines {
+		var (
+			attr = ""
+			info = &DeviceInfo{}
+		)
+
+		_, err := fmt.Sscan(line,
+			&info.Name,
+			&info.BlockDeviceName,
+			&attr,
+			&info.Major,
+			&info.Minor,
+			&info.OpenCount,
+			&info.TargetCount,
+			&info.EventNumber)
+
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse line %q", line)
+		}
+
+		// Parse attributes (see "man 8 dmsetup" for details)
+		info.Suspended = strings.Contains(attr, "s")
+		info.ReadOnly = strings.Contains(attr, "r")
+		info.TableLive = strings.Contains(attr, "L")
+		info.TableInactive = strings.Contains(attr, "I")
+
+		devices[i] = info
+	}
+
+	return devices, nil
+}
+
+// Version returns "dmsetup version" output
+func Version() (string, error) {
+	return dmsetup("version")
+}
+
+// GetFullDevicePath returns full path for the given device name (like "/dev/mapper/name")
+func GetFullDevicePath(deviceName string) string {
+	if strings.HasPrefix(deviceName, DevMapperDir) {
+		return deviceName
+	}
+
+	return DevMapperDir + deviceName
+}
+
+// BlockDeviceSize returns size of block device in bytes
+func BlockDeviceSize(devicePath string) (uint64, error) {
+	data, err := exec.Command("blockdev", "--getsize64", "-q", devicePath).CombinedOutput()
+	output := string(data)
+	if err != nil {
+		return 0, errors.Wrapf(err, output)
+	}
+
+	output = strings.TrimSuffix(output, "\n")
+	return strconv.ParseUint(output, 10, 64)
+}
+
+func dmsetup(args ...string) (string, error) {
+	data, err := exec.Command("dmsetup", args...).CombinedOutput()
+	output := string(data)
+	if err != nil {
+		// Try find Linux error code otherwise return generic error with dmsetup output
+		if errno, ok := tryGetUnixError(output); ok {
+			return "", errno
+		}
+
+		return "", errors.Wrapf(err, "dmsetup %s\nerror: %s\n", strings.Join(args, " "), output)
+	}
+
+	output = strings.TrimSuffix(output, "\n")
+	output = strings.TrimSpace(output)
+
+	return output, nil
+}
+
+// tryGetUnixError tries to find Linux error code from dmsetup output
+func tryGetUnixError(output string) (unix.Errno, bool) {
+	// It's useful to have Linux error codes like EBUSY, EPERM, ..., instead of just text.
+	// Unfortunately there is no better way than extracting/comparing error text.
+	text := parseDmsetupError(output)
+	if text == "" {
+		return 0, false
+	}
+
+	err, ok := errTable[text]
+	return err, ok
+}
+
+// dmsetup returns error messages in format:
+// 	device-mapper: message ioctl on <name> failed: File exists\n
+// 	Command failed\n
+// parseDmsetupError extracts text between "failed: " and "\n"
+func parseDmsetupError(output string) string {
+	lines := strings.SplitN(output, "\n", 2)
+	if len(lines) < 2 {
+		return ""
+	}
+
+	const failedSubstr = "failed: "
+
+	line := lines[0]
+	idx := strings.LastIndex(line, failedSubstr)
+	if idx == -1 {
+		return ""
+	}
+
+	str := line[idx:]
+
+	// Strip "failed: " prefix
+	str = strings.TrimPrefix(str, failedSubstr)
+
+	str = strings.ToLower(str)
+	return str
+}

snapshots/devmapper/dmsetup/dmsetup_test.go

@@ -0,0 +1,194 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package dmsetup
+
+import (
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/containerd/containerd/pkg/testutil"
+	"github.com/containerd/containerd/snapshots/devmapper/losetup"
+	"github.com/docker/go-units"
+	"golang.org/x/sys/unix"
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+)
+
+const (
+	testPoolName   = "test-pool"
+	testDeviceName = "test-device"
+	deviceID       = 1
+	snapshotID     = 2
+)
+
+func TestDMSetup(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	tempDir, err := ioutil.TempDir("", "dmsetup-tests-")
+	assert.NilError(t, err, "failed to make temp dir for tests")
+
+	defer func() {
+		err := os.RemoveAll(tempDir)
+		assert.NilError(t, err)
+	}()
+
+	dataImage, loopDataDevice := createLoopbackDevice(t, tempDir)
+	metaImage, loopMetaDevice := createLoopbackDevice(t, tempDir)
+
+	defer func() {
+		err = losetup.RemoveLoopDevicesAssociatedWithImage(dataImage)
+		assert.NilError(t, err, "failed to detach loop devices for data image: %s", dataImage)
+
+		err = losetup.RemoveLoopDevicesAssociatedWithImage(metaImage)
+		assert.NilError(t, err, "failed to detach loop devices for meta image: %s", metaImage)
+	}()
+
+	t.Run("CreatePool", func(t *testing.T) {
+		err := CreatePool(testPoolName, loopDataDevice, loopMetaDevice, 128)
+		assert.NilError(t, err, "failed to create thin-pool")
+
+		table, err := Table(testPoolName)
+		t.Logf("table: %s", table)
+		assert.NilError(t, err)
+		assert.Assert(t, strings.HasPrefix(table, "0 32768 thin-pool"))
+		assert.Assert(t, strings.HasSuffix(table, "128 32768 1 skip_block_zeroing"))
+	})
+
+	t.Run("ReloadPool", func(t *testing.T) {
+		err := ReloadPool(testPoolName, loopDataDevice, loopMetaDevice, 256)
+		assert.NilError(t, err, "failed to reload thin-pool")
+	})
+
+	t.Run("CreateDevice", testCreateDevice)
+
+	t.Run("CreateSnapshot", testCreateSnapshot)
+	t.Run("DeleteSnapshot", testDeleteSnapshot)
+
+	t.Run("ActivateDevice", testActivateDevice)
+	t.Run("SuspendResumeDevice", testSuspendResumeDevice)
+	t.Run("RemoveDevice", testRemoveDevice)
+
+	t.Run("RemovePool", func(t *testing.T) {
+		err = RemoveDevice(testPoolName, RemoveWithForce, RemoveWithRetries)
+		assert.NilError(t, err, "failed to remove thin-pool")
+	})
+
+	t.Run("Version", testVersion)
+}
+
+func testCreateDevice(t *testing.T) {
+	err := CreateDevice(testPoolName, deviceID)
+	assert.NilError(t, err, "failed to create test device")
+
+	err = CreateDevice(testPoolName, deviceID)
+	assert.Assert(t, err == unix.EEXIST)
+
+	infos, err := Info(testPoolName)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Len(infos, 1), "got unexpected number of device infos")
+}
+
+func testCreateSnapshot(t *testing.T) {
+	err := CreateSnapshot(testPoolName, snapshotID, deviceID)
+	assert.NilError(t, err)
+}
+
+func testDeleteSnapshot(t *testing.T) {
+	err := DeleteDevice(testPoolName, snapshotID)
+	assert.NilError(t, err, "failed to send delete message")
+
+	err = DeleteDevice(testPoolName, snapshotID)
+	assert.Assert(t, err == unix.ENODATA)
+}
+
+func testActivateDevice(t *testing.T) {
+	err := ActivateDevice(testPoolName, testDeviceName, 1, 1024, "")
+	assert.NilError(t, err, "failed to activate device")
+
+	err = ActivateDevice(testPoolName, testDeviceName, 1, 1024, "")
+	assert.Equal(t, err, unix.EBUSY)
+
+	if _, err := os.Stat("/dev/mapper/" + testDeviceName); err != nil && !os.IsExist(err) {
+		assert.NilError(t, err, "failed to stat device")
+	}
+
+	list, err := Info(testPoolName)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Len(list, 1))
+
+	info := list[0]
+	assert.Equal(t, testPoolName, info.Name)
+	assert.Assert(t, info.TableLive)
+}
+
+func testSuspendResumeDevice(t *testing.T) {
+	err := SuspendDevice(testDeviceName)
+	assert.NilError(t, err)
+
+	err = SuspendDevice(testDeviceName)
+	assert.NilError(t, err)
+
+	list, err := Info(testDeviceName)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Len(list, 1))
+
+	info := list[0]
+	assert.Assert(t, info.Suspended)
+
+	err = ResumeDevice(testDeviceName)
+	assert.NilError(t, err)
+
+	err = ResumeDevice(testDeviceName)
+	assert.NilError(t, err)
+}
+
+func testRemoveDevice(t *testing.T) {
+	err := RemoveDevice(testPoolName)
+	assert.Assert(t, err == unix.EBUSY, "removing thin-pool with dependencies shouldn't be allowed")
+
+	err = RemoveDevice(testDeviceName, RemoveWithRetries)
+	assert.NilError(t, err, "failed to remove thin-device")
+}
+
+func testVersion(t *testing.T) {
+	version, err := Version()
+	assert.NilError(t, err)
+	assert.Assert(t, version != "")
+}
+
+func createLoopbackDevice(t *testing.T, dir string) (string, string) {
+	file, err := ioutil.TempFile(dir, "dmsetup-tests-")
+	assert.NilError(t, err)
+
+	size, err := units.RAMInBytes("16Mb")
+	assert.NilError(t, err)
+
+	err = file.Truncate(size)
+	assert.NilError(t, err)
+
+	err = file.Close()
+	assert.NilError(t, err)
+
+	imagePath := file.Name()
+
+	loopDevice, err := losetup.AttachLoopDevice(imagePath)
+	assert.NilError(t, err)
+
+	return imagePath, loopDevice
+}