diff --git a/mount/losetup_linux.go b/mount/losetup_linux.go new file mode 100644 index 000000000..a0040dc18 --- /dev/null +++ b/mount/losetup_linux.go @@ -0,0 +1,196 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package mount + +import ( + "fmt" + "math/rand" + "os" + "strings" + "syscall" + "time" + "unsafe" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +const ( + loopControlPath = "/dev/loop-control" + loopDevFormat = "/dev/loop%d" + + ebusyString = "device or resource busy" +) + +// LoopParams parameters to control loop device setup +type LoopParams struct { + // Loop device should forbid write + Readonly bool + // Loop device is automatically cleared by kernel when the + // last opener closes it + Autoclear bool + // Use direct IO to access the loop backing file + Direct bool +} + +func ioctl(fd, req, args uintptr) (uintptr, uintptr, error) { + r1, r2, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, req, args) + if errno != 0 { + return 0, 0, errno + } + + return r1, r2, nil +} + +func getFreeLoopDev() (uint32, error) { + ctrl, err := os.OpenFile(loopControlPath, os.O_RDWR, 0) + if err != nil { + return 0, errors.Errorf("could not open %v: %v", loopControlPath, err) + } + defer ctrl.Close() + num, _, err := ioctl(ctrl.Fd(), unix.LOOP_CTL_GET_FREE, 0) + if err != nil { + return 0, errors.Wrap(err, "could not get free loop device") + } + return uint32(num), nil +} + +func setupLoopDev(backingFile, loopDev string, param LoopParams) error { + // 1. Open backing file and loop device + flags := os.O_RDWR + if param.Readonly { + flags = os.O_RDONLY + } + + back, err := os.OpenFile(backingFile, flags, 0) + if err != nil { + return errors.Wrapf(err, "could not open backing file: %s", backingFile) + } + defer back.Close() + + loop, err := os.OpenFile(loopDev, flags, 0) + if err != nil { + return errors.Wrapf(err, "could not open loop device: %s", loopDev) + } + defer loop.Close() + + // 2. Set FD + if _, _, err = ioctl(loop.Fd(), unix.LOOP_SET_FD, back.Fd()); err != nil { + return errors.Wrapf(err, "could not set loop fd for device: %s", loopDev) + } + + // 3. Set Info + info := unix.LoopInfo64{} + copy(info.File_name[:], backingFile) + if param.Readonly { + info.Flags |= unix.LO_FLAGS_READ_ONLY + } + + if param.Autoclear { + info.Flags |= unix.LO_FLAGS_AUTOCLEAR + } + + if param.Direct { + info.Flags |= unix.LO_FLAGS_DIRECT_IO + } + + _, _, err = ioctl(loop.Fd(), unix.LOOP_SET_STATUS64, uintptr(unsafe.Pointer(&info))) + if err == nil { + return nil + } + + if param.Direct { + // Retry w/o direct IO flag in case kernel does not support it. The downside is that + // it will suffer from double cache problem. + info.Flags &= ^(uint32(unix.LO_FLAGS_DIRECT_IO)) + _, _, err = ioctl(loop.Fd(), unix.LOOP_SET_STATUS64, uintptr(unsafe.Pointer(&info))) + if err == nil { + return nil + } + } + + // Cleanup loop fd and return error + _, _, _ = ioctl(loop.Fd(), unix.LOOP_CLR_FD, 0) + return errors.Errorf("failed to set loop device info: %v", err) +} + +// setupLoop looks for (and possibly creates) a free loop device, and +// then attaches backingFile to it. +// +// When autoclear is true, caller should take care to close it when +// done with the loop device. The loop device file handle keeps +// loFlagsAutoclear in effect and we rely on it to clean up the loop +// device. If caller closes the file handle after mounting the device, +// kernel will clear the loop device after it is umounted. Otherwise +// the loop device is cleared when the file handle is closed. +// +// When autoclear is false, caller should be responsible to remove +// the loop device when done with it. +// +// Upon success, the file handle to the loop device is returned. +func setupLoop(backingFile string, param LoopParams) (string, error) { + for retry := 1; retry < 100; retry++ { + num, err := getFreeLoopDev() + if err != nil { + return "", err + } + + loopDev := fmt.Sprintf(loopDevFormat, num) + if err := setupLoopDev(backingFile, loopDev, param); err != nil { + // Per util-linux/sys-utils/losetup.c:create_loop(), + // free loop device can race and we end up failing + // with EBUSY when trying to set it up. + if strings.Contains(err.Error(), ebusyString) { + // Fallback a bit to avoid live lock + time.Sleep(time.Millisecond * time.Duration(rand.Intn(retry*10))) + continue + } + return "", err + } + + return loopDev, nil + } + + return "", errors.New("timeout creating new loopback device") +} + +func removeLoop(loopdev string) error { + file, err := os.Open(loopdev) + if err != nil { + return err + } + defer file.Close() + + _, _, err = ioctl(file.Fd(), unix.LOOP_CLR_FD, 0) + return err +} + +// Attach a specified backing file to a loop device +func AttachLoopDevice(backingFile string) (string, error) { + return setupLoop(backingFile, LoopParams{}) +} + +// Detach a loop device +func DetachLoopDevice(devices ...string) error { + for _, dev := range devices { + if err := removeLoop(dev); err != nil { + return errors.Wrapf(err, "failed to remove loop device: %s", dev) + } + } + + return nil +} diff --git a/mount/losetup_linux_test.go b/mount/losetup_linux_test.go new file mode 100644 index 000000000..445bb05d0 --- /dev/null +++ b/mount/losetup_linux_test.go @@ -0,0 +1,115 @@ +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package mount + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/containerd/continuity/testutil" +) + +var randomData = []byte("randomdata") + +func createTempFile(t *testing.T) string { + t.Helper() + + f, err := ioutil.TempFile("", "losetup") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + if err = f.Truncate(512); err != nil { + t.Fatal(err) + } + + return f.Name() +} + +func TestNonExistingLoop(t *testing.T) { + testutil.RequiresRoot(t) + + backingFile := "setup-loop-test-no-such-file" + _, err := setupLoop(backingFile, LoopParams{}) + if err == nil { + t.Fatalf("setupLoop with non-existing file should fail") + } +} + +func TestRoLoop(t *testing.T) { + testutil.RequiresRoot(t) + + backingFile := createTempFile(t) + defer func() { + if err := os.Remove(backingFile); err != nil { + t.Fatal(err) + } + }() + + path, err := setupLoop(backingFile, LoopParams{Readonly: true, Autoclear: true}) + if err != nil { + t.Fatal(err) + } + + if err := ioutil.WriteFile(path, randomData, os.ModePerm); err == nil { + t.Fatalf("writing to readonly loop device should fail") + } +} + +func TestRwLoop(t *testing.T) { + testutil.RequiresRoot(t) + + backingFile := createTempFile(t) + defer func() { + if err := os.Remove(backingFile); err != nil { + t.Fatal(err) + } + }() + + path, err := setupLoop(backingFile, LoopParams{Autoclear: true}) + if err != nil { + t.Fatal(err) + } + + if err := ioutil.WriteFile(path, randomData, os.ModePerm); err != nil { + t.Fatal(err) + } +} + +func TestAttachDetachLoopDevice(t *testing.T) { + testutil.RequiresRoot(t) + + path := createTempFile(t) + defer func() { + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + }() + + dev, err := AttachLoopDevice(path) + if err != nil { + t.Fatal(err) + } + + if err = DetachLoopDevice(dev); err != nil { + t.Fatal(err) + } +} diff --git a/mount/mount_linux.go b/mount/mount_linux.go index 6d28f0a97..7191d40b1 100644 --- a/mount/mount_linux.go +++ b/mount/mount_linux.go @@ -42,7 +42,7 @@ func init() { // // If m.Type starts with "fuse." or "fuse3.", "mount.fuse" or "mount.fuse3" // helper binary is called. -func (m *Mount) Mount(target string) error { +func (m *Mount) Mount(target string) (err error) { for _, helperBinary := range allowedHelperBinaries { // helperBinary = "mount.fuse", typePrefix = "fuse." typePrefix := strings.TrimPrefix(helperBinary, "mount.") + "." @@ -62,7 +62,7 @@ func (m *Mount) Mount(target string) error { chdir, options = compactLowerdirOption(options) } - flags, data := parseMountOptions(options) + flags, data, losetup := parseMountOptions(options) if len(data) > pagesize { return errors.Errorf("mount options is too long") } @@ -77,7 +77,18 @@ func (m *Mount) Mount(target string) error { if flags&unix.MS_REMOUNT == 0 || data != "" { // Initial call applying all non-propagation flags for mount // or remount with changed data - if err := mountAt(chdir, m.Source, target, m.Type, uintptr(oflags), data); err != nil { + source := m.Source + if losetup { + devFile, err := setupLoop(m.Source, LoopParams{ + Readonly: oflags&unix.MS_RDONLY == unix.MS_RDONLY, + Autoclear: true}) + if err != nil { + return err + } + // Mount the loop device instead + source = devFile + } + if err := mountAt(chdir, source, target, m.Type, uintptr(oflags), data); err != nil { return err } } @@ -186,11 +197,13 @@ func UnmountAll(mount string, flags int) error { // parseMountOptions takes fstab style mount options and parses them for // use with a standard mount() syscall -func parseMountOptions(options []string) (int, string) { +func parseMountOptions(options []string) (int, string, bool) { var ( - flag int - data []string + flag int + losetup bool + data []string ) + loopOpt := "loop" flags := map[string]struct { clear bool flag int @@ -231,11 +244,13 @@ func parseMountOptions(options []string) (int, string) { } else { flag |= f.flag } + } else if o == loopOpt { + losetup = true } else { data = append(data, o) } } - return flag, strings.Join(data, ",") + return flag, strings.Join(data, ","), losetup } // compactLowerdirOption updates overlay lowdir option and returns the common diff --git a/snapshots/devmapper/dmsetup/dmsetup_test.go b/snapshots/devmapper/dmsetup/dmsetup_test.go index 3b4c688bb..e8b95330b 100644 --- a/snapshots/devmapper/dmsetup/dmsetup_test.go +++ b/snapshots/devmapper/dmsetup/dmsetup_test.go @@ -29,8 +29,8 @@ import ( "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" + "github.com/containerd/containerd/mount" "github.com/containerd/containerd/pkg/testutil" - "github.com/containerd/containerd/snapshots/devmapper/losetup" ) const ( @@ -55,16 +55,13 @@ func TestDMSetup(t *testing.T) { metaImage, loopMetaDevice := createLoopbackDevice(t, tempDir) defer func() { - err = losetup.RemoveLoopDevicesAssociatedWithImage(dataImage) - assert.NilError(t, err, "failed to detach loop devices for data image: %s", dataImage) - - err = losetup.RemoveLoopDevicesAssociatedWithImage(metaImage) - assert.NilError(t, err, "failed to detach loop devices for meta image: %s", metaImage) + err = mount.DetachLoopDevice(loopDataDevice, loopMetaDevice) + assert.NilError(t, err, "failed to detach loop devices for data image: %s and meta image: %s", dataImage, metaImage) }() t.Run("CreatePool", func(t *testing.T) { err := CreatePool(testPoolName, loopDataDevice, loopMetaDevice, 128) - assert.NilError(t, err, "failed to create thin-pool") + assert.NilError(t, err, "failed to create thin-pool with %s %s", loopDataDevice, loopMetaDevice) table, err := Table(testPoolName) t.Logf("table: %s", table) @@ -201,7 +198,7 @@ func createLoopbackDevice(t *testing.T, dir string) (string, string) { imagePath := file.Name() - loopDevice, err := losetup.AttachLoopDevice(imagePath) + loopDevice, err := mount.AttachLoopDevice(imagePath) assert.NilError(t, err) return imagePath, loopDevice diff --git a/snapshots/devmapper/losetup/losetup.go b/snapshots/devmapper/losetup/losetup.go deleted file mode 100644 index b9e9bf980..000000000 --- a/snapshots/devmapper/losetup/losetup.go +++ /dev/null @@ -1,92 +0,0 @@ -// +build linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package losetup - -import ( - "os/exec" - "strings" - - "github.com/pkg/errors" - "golang.org/x/sys/unix" -) - -// FindAssociatedLoopDevices returns a list of loop devices attached to a given image -func FindAssociatedLoopDevices(imagePath string) ([]string, error) { - output, err := losetup("--list", "--output", "NAME", "--associated", imagePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to get loop devices: '%s'", output) - } - - if output == "" { - return []string{}, nil - } - - items := strings.Split(output, "\n") - if len(items) <= 1 { - return []string{}, nil - } - - // Skip header with column names - return items[1:], nil -} - -// AttachLoopDevice finds first available loop device and associates it with an image. -func AttachLoopDevice(imagePath string) (string, error) { - return losetup("--find", "--show", imagePath) -} - -// DetachLoopDevice detaches loop devices -func DetachLoopDevice(loopDevice ...string) error { - args := append([]string{"--detach"}, loopDevice...) - _, err := losetup(args...) - return err -} - -// RemoveLoopDevicesAssociatedWithImage detaches all loop devices attached to a given sparse image -func RemoveLoopDevicesAssociatedWithImage(imagePath string) error { - loopDevices, err := FindAssociatedLoopDevices(imagePath) - if err != nil { - return err - } - - for _, loopDevice := range loopDevices { - if err = DetachLoopDevice(loopDevice); err != nil && err != unix.ENOENT { - return err - } - } - - return nil -} - -// losetup is a wrapper around losetup command line tool -func losetup(args ...string) (string, error) { - cmd := exec.Command("losetup", args...) - cmd.Env = append(cmd.Env, "LANG=C") - data, err := cmd.CombinedOutput() - output := string(data) - if err != nil { - if strings.Contains(output, "No such file or directory") || strings.Contains(output, "No such device") { - return "", unix.ENOENT - } - - return "", errors.Wrapf(err, "losetup %s\nerror: %s\n", strings.Join(args, " "), output) - } - - return strings.TrimSuffix(output, "\n"), err -} diff --git a/snapshots/devmapper/losetup/losetup_test.go b/snapshots/devmapper/losetup/losetup_test.go deleted file mode 100644 index b6d2d5ebc..000000000 --- a/snapshots/devmapper/losetup/losetup_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// +build linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package losetup - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/docker/go-units" - "golang.org/x/sys/unix" - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" - - "github.com/containerd/containerd/pkg/testutil" -) - -func TestLosetup(t *testing.T) { - testutil.RequiresRoot(t) - - var ( - imagePath = createSparseImage(t) - loopDevice1 string - loopDevice2 string - ) - - defer func() { - err := os.Remove(imagePath) - assert.NilError(t, err) - }() - - t.Run("AttachLoopDevice", func(t *testing.T) { - dev1, err := AttachLoopDevice(imagePath) - assert.NilError(t, err) - assert.Assert(t, dev1 != "") - - dev2, err := AttachLoopDevice(imagePath) - assert.NilError(t, err) - assert.Assert(t, dev2 != dev1, "should attach different loop device") - - loopDevice1 = dev1 - loopDevice2 = dev2 - }) - - t.Run("AttachEmptyLoopDevice", func(t *testing.T) { - _, err := AttachLoopDevice("") - assert.Assert(t, err != nil, "shouldn't attach empty path") - }) - - t.Run("FindAssociatedLoopDevices", func(t *testing.T) { - devices, err := FindAssociatedLoopDevices(imagePath) - assert.NilError(t, err) - assert.Assert(t, is.Len(devices, 2), "unexpected number of attached devices") - assert.Assert(t, is.Contains(devices, loopDevice1)) - assert.Assert(t, is.Contains(devices, loopDevice2)) - }) - - t.Run("FindAssociatedLoopDevicesForInvalidImage", func(t *testing.T) { - devices, err := FindAssociatedLoopDevices("") - assert.NilError(t, err) - assert.Assert(t, is.Len(devices, 0)) - }) - - t.Run("DetachLoopDevice", func(t *testing.T) { - err := DetachLoopDevice(loopDevice2) - assert.NilError(t, err, "failed to detach %q", loopDevice2) - }) - - t.Run("DetachEmptyDevice", func(t *testing.T) { - err := DetachLoopDevice("") - assert.Assert(t, err != nil, "shouldn't detach empty path") - }) - - t.Run("RemoveLoopDevicesAssociatedWithImage", func(t *testing.T) { - err := RemoveLoopDevicesAssociatedWithImage(imagePath) - assert.NilError(t, err) - - devices, err := FindAssociatedLoopDevices(imagePath) - assert.NilError(t, err) - assert.Assert(t, is.Len(devices, 0)) - }) - - t.Run("RemoveLoopDevicesAssociatedWithInvalidImage", func(t *testing.T) { - err := RemoveLoopDevicesAssociatedWithImage("") - assert.NilError(t, err) - }) - - t.Run("DetachInvalidDevice", func(t *testing.T) { - err := DetachLoopDevice("/dev/loop_invalid_idx") - assert.Equal(t, unix.ENOENT, err) - }) -} - -func createSparseImage(t *testing.T) string { - file, err := ioutil.TempFile("", "losetup-tests-") - assert.NilError(t, err) - - size, err := units.RAMInBytes("16Mb") - assert.NilError(t, err) - - err = file.Truncate(size) - assert.NilError(t, err) - - err = file.Close() - assert.NilError(t, err) - - return file.Name() -} diff --git a/snapshots/devmapper/pool_device_test.go b/snapshots/devmapper/pool_device_test.go index cc729a3a0..8525c0028 100644 --- a/snapshots/devmapper/pool_device_test.go +++ b/snapshots/devmapper/pool_device_test.go @@ -31,7 +31,6 @@ import ( "github.com/containerd/containerd/mount" "github.com/containerd/containerd/pkg/testutil" "github.com/containerd/containerd/snapshots/devmapper/dmsetup" - "github.com/containerd/containerd/snapshots/devmapper/losetup" "github.com/docker/go-units" "github.com/sirupsen/logrus" "gotest.tools/v3/assert" @@ -74,7 +73,7 @@ func TestPoolDevice(t *testing.T) { defer func() { // Detach loop devices and remove images - err := losetup.DetachLoopDevice(loopDataDevice, loopMetaDevice) + err := mount.DetachLoopDevice(loopDataDevice, loopMetaDevice) assert.NilError(t, err) err = os.RemoveAll(tempDir) @@ -306,7 +305,7 @@ func createLoopbackDevice(t *testing.T, dir string) (string, string) { imagePath := file.Name() - loopDevice, err := losetup.AttachLoopDevice(imagePath) + loopDevice, err := mount.AttachLoopDevice(imagePath) assert.NilError(t, err) return imagePath, loopDevice diff --git a/snapshots/devmapper/snapshotter_test.go b/snapshots/devmapper/snapshotter_test.go index 5157be869..419ed7206 100644 --- a/snapshots/devmapper/snapshotter_test.go +++ b/snapshots/devmapper/snapshotter_test.go @@ -37,7 +37,6 @@ import ( "github.com/containerd/containerd/pkg/testutil" "github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots/devmapper/dmsetup" - "github.com/containerd/containerd/snapshots/devmapper/losetup" "github.com/containerd/containerd/snapshots/testsuite" ) @@ -70,7 +69,7 @@ func TestSnapshotterSuite(t *testing.T) { removePool := func() error { result := multierror.Append( snap.pool.RemovePool(ctx), - losetup.DetachLoopDevice(loopDataDevice, loopMetaDevice)) + mount.DetachLoopDevice(loopDataDevice, loopMetaDevice)) return result.ErrorOrNil() }