Merge pull request #4902 from mxpv/losetup

[Carry] mount: handle loopback mount

mount/losetup_linux.go

@@ -0,0 +1,196 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"strings"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	loopControlPath = "/dev/loop-control"
+	loopDevFormat   = "/dev/loop%d"
+
+	ebusyString = "device or resource busy"
+)
+
+// LoopParams parameters to control loop device setup
+type LoopParams struct {
+	// Loop device should forbid write
+	Readonly bool
+	// Loop device is automatically cleared by kernel when the
+	// last opener closes it
+	Autoclear bool
+	// Use direct IO to access the loop backing file
+	Direct bool
+}
+
+func ioctl(fd, req, args uintptr) (uintptr, uintptr, error) {
+	r1, r2, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, req, args)
+	if errno != 0 {
+		return 0, 0, errno
+	}
+
+	return r1, r2, nil
+}
+
+func getFreeLoopDev() (uint32, error) {
+	ctrl, err := os.OpenFile(loopControlPath, os.O_RDWR, 0)
+	if err != nil {
+		return 0, errors.Errorf("could not open %v: %v", loopControlPath, err)
+	}
+	defer ctrl.Close()
+	num, _, err := ioctl(ctrl.Fd(), unix.LOOP_CTL_GET_FREE, 0)
+	if err != nil {
+		return 0, errors.Wrap(err, "could not get free loop device")
+	}
+	return uint32(num), nil
+}
+
+func setupLoopDev(backingFile, loopDev string, param LoopParams) error {
+	// 1. Open backing file and loop device
+	flags := os.O_RDWR
+	if param.Readonly {
+		flags = os.O_RDONLY
+	}
+
+	back, err := os.OpenFile(backingFile, flags, 0)
+	if err != nil {
+		return errors.Wrapf(err, "could not open backing file: %s", backingFile)
+	}
+	defer back.Close()
+
+	loop, err := os.OpenFile(loopDev, flags, 0)
+	if err != nil {
+		return errors.Wrapf(err, "could not open loop device: %s", loopDev)
+	}
+	defer loop.Close()
+
+	// 2. Set FD
+	if _, _, err = ioctl(loop.Fd(), unix.LOOP_SET_FD, back.Fd()); err != nil {
+		return errors.Wrapf(err, "could not set loop fd for device: %s", loopDev)
+	}
+
+	// 3. Set Info
+	info := unix.LoopInfo64{}
+	copy(info.File_name[:], backingFile)
+	if param.Readonly {
+		info.Flags |= unix.LO_FLAGS_READ_ONLY
+	}
+
+	if param.Autoclear {
+		info.Flags |= unix.LO_FLAGS_AUTOCLEAR
+	}
+
+	if param.Direct {
+		info.Flags |= unix.LO_FLAGS_DIRECT_IO
+	}
+
+	_, _, err = ioctl(loop.Fd(), unix.LOOP_SET_STATUS64, uintptr(unsafe.Pointer(&info)))
+	if err == nil {
+		return nil
+	}
+
+	if param.Direct {
+		// Retry w/o direct IO flag in case kernel does not support it. The downside is that
+		// it will suffer from double cache problem.
+		info.Flags &= ^(uint32(unix.LO_FLAGS_DIRECT_IO))
+		_, _, err = ioctl(loop.Fd(), unix.LOOP_SET_STATUS64, uintptr(unsafe.Pointer(&info)))
+		if err == nil {
+			return nil
+		}
+	}
+
+	// Cleanup loop fd and return error
+	_, _, _ = ioctl(loop.Fd(), unix.LOOP_CLR_FD, 0)
+	return errors.Errorf("failed to set loop device info: %v", err)
+}
+
+// setupLoop looks for (and possibly creates) a free loop device, and
+// then attaches backingFile to it.
+//
+// When autoclear is true, caller should take care to close it when
+// done with the loop device. The loop device file handle keeps
+// loFlagsAutoclear in effect and we rely on it to clean up the loop
+// device. If caller closes the file handle after mounting the device,
+// kernel will clear the loop device after it is umounted. Otherwise
+// the loop device is cleared when the file handle is closed.
+//
+// When autoclear is false, caller should be responsible to remove
+// the loop device when done with it.
+//
+// Upon success, the file handle to the loop device is returned.
+func setupLoop(backingFile string, param LoopParams) (string, error) {
+	for retry := 1; retry < 100; retry++ {
+		num, err := getFreeLoopDev()
+		if err != nil {
+			return "", err
+		}
+
+		loopDev := fmt.Sprintf(loopDevFormat, num)
+		if err := setupLoopDev(backingFile, loopDev, param); err != nil {
+			// Per util-linux/sys-utils/losetup.c:create_loop(),
+			// free loop device can race and we end up failing
+			// with EBUSY when trying to set it up.
+			if strings.Contains(err.Error(), ebusyString) {
+				// Fallback a bit to avoid live lock
+				time.Sleep(time.Millisecond * time.Duration(rand.Intn(retry*10)))
+				continue
+			}
+			return "", err
+		}
+
+		return loopDev, nil
+	}
+
+	return "", errors.New("timeout creating new loopback device")
+}
+
+func removeLoop(loopdev string) error {
+	file, err := os.Open(loopdev)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, _, err = ioctl(file.Fd(), unix.LOOP_CLR_FD, 0)
+	return err
+}
+
+// Attach a specified backing file to a loop device
+func AttachLoopDevice(backingFile string) (string, error) {
+	return setupLoop(backingFile, LoopParams{})
+}
+
+// Detach a loop device
+func DetachLoopDevice(devices ...string) error {
+	for _, dev := range devices {
+		if err := removeLoop(dev); err != nil {
+			return errors.Wrapf(err, "failed to remove loop device: %s", dev)
+		}
+	}
+
+	return nil
+}

mount/losetup_linux_test.go

@@ -0,0 +1,115 @@
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package mount
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/containerd/continuity/testutil"
+)
+
+var randomData = []byte("randomdata")
+
+func createTempFile(t *testing.T) string {
+	t.Helper()
+
+	f, err := ioutil.TempFile("", "losetup")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	if err = f.Truncate(512); err != nil {
+		t.Fatal(err)
+	}
+
+	return f.Name()
+}
+
+func TestNonExistingLoop(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	backingFile := "setup-loop-test-no-such-file"
+	_, err := setupLoop(backingFile, LoopParams{})
+	if err == nil {
+		t.Fatalf("setupLoop with non-existing file should fail")
+	}
+}
+
+func TestRoLoop(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	backingFile := createTempFile(t)
+	defer func() {
+		if err := os.Remove(backingFile); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	path, err := setupLoop(backingFile, LoopParams{Readonly: true, Autoclear: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(path, randomData, os.ModePerm); err == nil {
+		t.Fatalf("writing to readonly loop device should fail")
+	}
+}
+
+func TestRwLoop(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	backingFile := createTempFile(t)
+	defer func() {
+		if err := os.Remove(backingFile); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	path, err := setupLoop(backingFile, LoopParams{Autoclear: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(path, randomData, os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAttachDetachLoopDevice(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	path := createTempFile(t)
+	defer func() {
+		if err := os.Remove(path); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	dev, err := AttachLoopDevice(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = DetachLoopDevice(dev); err != nil {
+		t.Fatal(err)
+	}
+}

mount/mount_linux.go

@@ -42,7 +42,7 @@ func init() {
 //
 // If m.Type starts with "fuse." or "fuse3.", "mount.fuse" or "mount.fuse3"
 // helper binary is called.
-func (m *Mount) Mount(target string) error {
+func (m *Mount) Mount(target string) (err error) {
 	for _, helperBinary := range allowedHelperBinaries {
 		// helperBinary = "mount.fuse", typePrefix = "fuse."
 		typePrefix := strings.TrimPrefix(helperBinary, "mount.") + "."
@@ -62,7 +62,7 @@ func (m *Mount) Mount(target string) error {
 		chdir, options = compactLowerdirOption(options)
 	}
 
-	flags, data := parseMountOptions(options)
+	flags, data, losetup := parseMountOptions(options)
 	if len(data) > pagesize {
 		return errors.Errorf("mount options is too long")
 	}
@@ -77,7 +77,18 @@ func (m *Mount) Mount(target string) error {
 	if flags&unix.MS_REMOUNT == 0 || data != "" {
 		// Initial call applying all non-propagation flags for mount
 		// or remount with changed data
-		if err := mountAt(chdir, m.Source, target, m.Type, uintptr(oflags), data); err != nil {
+		source := m.Source
+		if losetup {
+			devFile, err := setupLoop(m.Source, LoopParams{
+				Readonly:  oflags&unix.MS_RDONLY == unix.MS_RDONLY,
+				Autoclear: true})
+			if err != nil {
+				return err
+			}
+			// Mount the loop device instead
+			source = devFile
+		}
+		if err := mountAt(chdir, source, target, m.Type, uintptr(oflags), data); err != nil {
 			return err
 		}
 	}
@@ -186,11 +197,13 @@ func UnmountAll(mount string, flags int) error {
 
 // parseMountOptions takes fstab style mount options and parses them for
 // use with a standard mount() syscall
-func parseMountOptions(options []string) (int, string) {
+func parseMountOptions(options []string) (int, string, bool) {
 	var (
 		flag    int
+		losetup bool
 		data    []string
 	)
+	loopOpt := "loop"
 	flags := map[string]struct {
 		clear bool
 		flag  int
@@ -231,11 +244,13 @@ func parseMountOptions(options []string) (int, string) {
 			} else {
 				flag |= f.flag
 			}
+		} else if o == loopOpt {
+			losetup = true
 		} else {
 			data = append(data, o)
 		}
 	}
-	return flag, strings.Join(data, ",")
+	return flag, strings.Join(data, ","), losetup
 }
 
 // compactLowerdirOption updates overlay lowdir option and returns the common

snapshots/devmapper/dmsetup/dmsetup_test.go

@@ -29,8 +29,8 @@ import (
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 
+	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/pkg/testutil"
-	"github.com/containerd/containerd/snapshots/devmapper/losetup"
 )
 
 const (
@@ -55,16 +55,13 @@ func TestDMSetup(t *testing.T) {
 	metaImage, loopMetaDevice := createLoopbackDevice(t, tempDir)
 
 	defer func() {
-		err = losetup.RemoveLoopDevicesAssociatedWithImage(dataImage)
+		err = mount.DetachLoopDevice(loopDataDevice, loopMetaDevice)
-		assert.NilError(t, err, "failed to detach loop devices for data image: %s", dataImage)
+		assert.NilError(t, err, "failed to detach loop devices for data image: %s and meta image: %s", dataImage, metaImage)
-
-		err = losetup.RemoveLoopDevicesAssociatedWithImage(metaImage)
-		assert.NilError(t, err, "failed to detach loop devices for meta image: %s", metaImage)
 	}()
 
 	t.Run("CreatePool", func(t *testing.T) {
 		err := CreatePool(testPoolName, loopDataDevice, loopMetaDevice, 128)
-		assert.NilError(t, err, "failed to create thin-pool")
+		assert.NilError(t, err, "failed to create thin-pool with %s %s", loopDataDevice, loopMetaDevice)
 
 		table, err := Table(testPoolName)
 		t.Logf("table: %s", table)
@@ -201,7 +198,7 @@ func createLoopbackDevice(t *testing.T, dir string) (string, string) {
 
 	imagePath := file.Name()
 
-	loopDevice, err := losetup.AttachLoopDevice(imagePath)
+	loopDevice, err := mount.AttachLoopDevice(imagePath)
 	assert.NilError(t, err)
 
 	return imagePath, loopDevice

snapshots/devmapper/losetup/losetup.go

@@ -1,92 +0,0 @@
-// +build linux
-
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package losetup
-
-import (
-	"os/exec"
-	"strings"
-
-	"github.com/pkg/errors"
-	"golang.org/x/sys/unix"
-)
-
-// FindAssociatedLoopDevices returns a list of loop devices attached to a given image
-func FindAssociatedLoopDevices(imagePath string) ([]string, error) {
-	output, err := losetup("--list", "--output", "NAME", "--associated", imagePath)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to get loop devices: '%s'", output)
-	}
-
-	if output == "" {
-		return []string{}, nil
-	}
-
-	items := strings.Split(output, "\n")
-	if len(items) <= 1 {
-		return []string{}, nil
-	}
-
-	// Skip header with column names
-	return items[1:], nil
-}
-
-// AttachLoopDevice finds first available loop device and associates it with an image.
-func AttachLoopDevice(imagePath string) (string, error) {
-	return losetup("--find", "--show", imagePath)
-}
-
-// DetachLoopDevice detaches loop devices
-func DetachLoopDevice(loopDevice ...string) error {
-	args := append([]string{"--detach"}, loopDevice...)
-	_, err := losetup(args...)
-	return err
-}
-
-// RemoveLoopDevicesAssociatedWithImage detaches all loop devices attached to a given sparse image
-func RemoveLoopDevicesAssociatedWithImage(imagePath string) error {
-	loopDevices, err := FindAssociatedLoopDevices(imagePath)
-	if err != nil {
-		return err
-	}
-
-	for _, loopDevice := range loopDevices {
-		if err = DetachLoopDevice(loopDevice); err != nil && err != unix.ENOENT {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// losetup is a wrapper around losetup command line tool
-func losetup(args ...string) (string, error) {
-	cmd := exec.Command("losetup", args...)
-	cmd.Env = append(cmd.Env, "LANG=C")
-	data, err := cmd.CombinedOutput()
-	output := string(data)
-	if err != nil {
-		if strings.Contains(output, "No such file or directory") || strings.Contains(output, "No such device") {
-			return "", unix.ENOENT
-		}
-
-		return "", errors.Wrapf(err, "losetup %s\nerror: %s\n", strings.Join(args, " "), output)
-	}
-
-	return strings.TrimSuffix(output, "\n"), err
-}

snapshots/devmapper/losetup/losetup_test.go

@@ -1,124 +0,0 @@
-// +build linux
-
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package losetup
-
-import (
-	"io/ioutil"
-	"os"
-	"testing"
-
-	"github.com/docker/go-units"
-	"golang.org/x/sys/unix"
-	"gotest.tools/v3/assert"
-	is "gotest.tools/v3/assert/cmp"
-
-	"github.com/containerd/containerd/pkg/testutil"
-)
-
-func TestLosetup(t *testing.T) {
-	testutil.RequiresRoot(t)
-
-	var (
-		imagePath   = createSparseImage(t)
-		loopDevice1 string
-		loopDevice2 string
-	)
-
-	defer func() {
-		err := os.Remove(imagePath)
-		assert.NilError(t, err)
-	}()
-
-	t.Run("AttachLoopDevice", func(t *testing.T) {
-		dev1, err := AttachLoopDevice(imagePath)
-		assert.NilError(t, err)
-		assert.Assert(t, dev1 != "")
-
-		dev2, err := AttachLoopDevice(imagePath)
-		assert.NilError(t, err)
-		assert.Assert(t, dev2 != dev1, "should attach different loop device")
-
-		loopDevice1 = dev1
-		loopDevice2 = dev2
-	})
-
-	t.Run("AttachEmptyLoopDevice", func(t *testing.T) {
-		_, err := AttachLoopDevice("")
-		assert.Assert(t, err != nil, "shouldn't attach empty path")
-	})
-
-	t.Run("FindAssociatedLoopDevices", func(t *testing.T) {
-		devices, err := FindAssociatedLoopDevices(imagePath)
-		assert.NilError(t, err)
-		assert.Assert(t, is.Len(devices, 2), "unexpected number of attached devices")
-		assert.Assert(t, is.Contains(devices, loopDevice1))
-		assert.Assert(t, is.Contains(devices, loopDevice2))
-	})
-
-	t.Run("FindAssociatedLoopDevicesForInvalidImage", func(t *testing.T) {
-		devices, err := FindAssociatedLoopDevices("")
-		assert.NilError(t, err)
-		assert.Assert(t, is.Len(devices, 0))
-	})
-
-	t.Run("DetachLoopDevice", func(t *testing.T) {
-		err := DetachLoopDevice(loopDevice2)
-		assert.NilError(t, err, "failed to detach %q", loopDevice2)
-	})
-
-	t.Run("DetachEmptyDevice", func(t *testing.T) {
-		err := DetachLoopDevice("")
-		assert.Assert(t, err != nil, "shouldn't detach empty path")
-	})
-
-	t.Run("RemoveLoopDevicesAssociatedWithImage", func(t *testing.T) {
-		err := RemoveLoopDevicesAssociatedWithImage(imagePath)
-		assert.NilError(t, err)
-
-		devices, err := FindAssociatedLoopDevices(imagePath)
-		assert.NilError(t, err)
-		assert.Assert(t, is.Len(devices, 0))
-	})
-
-	t.Run("RemoveLoopDevicesAssociatedWithInvalidImage", func(t *testing.T) {
-		err := RemoveLoopDevicesAssociatedWithImage("")
-		assert.NilError(t, err)
-	})
-
-	t.Run("DetachInvalidDevice", func(t *testing.T) {
-		err := DetachLoopDevice("/dev/loop_invalid_idx")
-		assert.Equal(t, unix.ENOENT, err)
-	})
-}
-
-func createSparseImage(t *testing.T) string {
-	file, err := ioutil.TempFile("", "losetup-tests-")
-	assert.NilError(t, err)
-
-	size, err := units.RAMInBytes("16Mb")
-	assert.NilError(t, err)
-
-	err = file.Truncate(size)
-	assert.NilError(t, err)
-
-	err = file.Close()
-	assert.NilError(t, err)
-
-	return file.Name()
-}

snapshots/devmapper/pool_device_test.go

@@ -31,7 +31,6 @@ import (
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/pkg/testutil"
 	"github.com/containerd/containerd/snapshots/devmapper/dmsetup"
-	"github.com/containerd/containerd/snapshots/devmapper/losetup"
 	"github.com/docker/go-units"
 	"github.com/sirupsen/logrus"
 	"gotest.tools/v3/assert"
@@ -74,7 +73,7 @@ func TestPoolDevice(t *testing.T) {
 
 	defer func() {
 		// Detach loop devices and remove images
-		err := losetup.DetachLoopDevice(loopDataDevice, loopMetaDevice)
+		err := mount.DetachLoopDevice(loopDataDevice, loopMetaDevice)
 		assert.NilError(t, err)
 
 		err = os.RemoveAll(tempDir)
@@ -306,7 +305,7 @@ func createLoopbackDevice(t *testing.T, dir string) (string, string) {
 
 	imagePath := file.Name()
 
-	loopDevice, err := losetup.AttachLoopDevice(imagePath)
+	loopDevice, err := mount.AttachLoopDevice(imagePath)
 	assert.NilError(t, err)
 
 	return imagePath, loopDevice

snapshots/devmapper/snapshotter_test.go

@@ -37,7 +37,6 @@ import (
 	"github.com/containerd/containerd/pkg/testutil"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/containerd/snapshots/devmapper/dmsetup"
-	"github.com/containerd/containerd/snapshots/devmapper/losetup"
 	"github.com/containerd/containerd/snapshots/testsuite"
 )
 
@@ -70,7 +69,7 @@ func TestSnapshotterSuite(t *testing.T) {
 		removePool := func() error {
 			result := multierror.Append(
 				snap.pool.RemovePool(ctx),
-				losetup.DetachLoopDevice(loopDataDevice, loopMetaDevice))
+				mount.DetachLoopDevice(loopDataDevice, loopMetaDevice))
 
 			return result.ErrorOrNil()
 		}