From 9e4207016903d274900c0585405cc0bfcfd2e4ef Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Fri, 13 Mar 2020 14:40:58 +0800 Subject: [PATCH] mount: handle loopback mount If a mount has specified `loop` option, we need to handle it on our own instead of passing it to the kernel. In such case, create a loopback device, attach the mount source to it, and mount the loopback device rather than the mount source. Signed-off-by: Peng Tao --- mount/losetup_linux.go | 230 ++++++++++++++++++++++++++++++++++++ mount/losetup_linux_test.go | 117 ++++++++++++++++++ mount/mount_linux.go | 30 +++-- 3 files changed, 370 insertions(+), 7 deletions(-) create mode 100644 mount/losetup_linux.go create mode 100644 mount/losetup_linux_test.go diff --git a/mount/losetup_linux.go b/mount/losetup_linux.go new file mode 100644 index 000000000..68fd799b0 --- /dev/null +++ b/mount/losetup_linux.go @@ -0,0 +1,230 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package mount + +import ( + "fmt" + "math/rand" + "os" + "strings" + "syscall" + "time" + "unsafe" + + "github.com/pkg/errors" +) + +const ( + loopControlPath = "/dev/loop-control" + loopDevFormat = "/dev/loop%d" + + // According to util-linux/include/loopdev.h + ioctlSetFd = 0x4C00 + ioctlClrFd = 0x4C01 + ioctlSetStatus64 = 0x4C04 + ioctlGetFree = 0x4C82 + + loFlagsReadonly = 1 + //loFlagsUseAops = 2 + loFlagsAutoclear = 4 + //loFlagsPartScan = 8 + loFlagsDirectIO = 16 + + ebusyString = "device or resource busy" +) + +// parameters to control loop device setup +type LoopParams struct { + // Loop device should forbid write + Readonly bool + // Loop device is automatically cleared by kernel when the + // last opener closes it + Autoclear bool + // Use direct IO to access the loop backing file + Direct bool +} + +// struct loop_info64 in util-linux/include/loopdev.h +type loopInfo struct { + /* + device uint64 + inode uint64 + rdevice uint64 + offset uint64 + sizelimit uint64 + number uint32 + encryptType uint32 + encryptKeySize uint32 + */ + _ [13]uint32 + flags uint32 + fileName [64]byte + /* + cryptName [64]byte + encryptKey [32]byte + init [2]uint64 + */ + _ [112]byte +} + +func ioctl(fd, req, args uintptr) (uintptr, uintptr, error) { + r1, r2, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, req, args) + if errno != 0 { + return 0, 0, errno + } + + return r1, r2, nil +} + +func getFreeLoopDev() (uint32, error) { + ctrl, err := os.OpenFile(loopControlPath, os.O_RDWR, 0) + if err != nil { + return 0, errors.Errorf("could not open %v: %v", loopControlPath, err) + } + defer ctrl.Close() + num, _, err := ioctl(ctrl.Fd(), ioctlGetFree, 0) + if err != nil { + return 0, errors.Wrap(err, "could not get free loop device") + } + return uint32(num), nil +} + +func setupLoopDev(backingFile, loopDev string, param LoopParams) (devFile *os.File, err error) { + // 1. Open backing file and loop device + oflags := os.O_RDWR + if param.Readonly { + oflags = os.O_RDONLY + } + back, err := os.OpenFile(backingFile, oflags, 0) + if err != nil { + return nil, errors.Errorf("could not open backing file: %v", err) + } + defer back.Close() + + loopFile, err := os.OpenFile(loopDev, oflags, 0) + if err != nil { + return nil, errors.Errorf("could not open loop device: %v", err) + } + defer func() { + if err != nil { + loopFile.Close() + } + }() + + // 2. Set FD + if _, _, err = ioctl(loopFile.Fd(), ioctlSetFd, back.Fd()); err != nil { + return nil, errors.Errorf("could not set loop fd: %v", err) + } + + // 3. Set Info + info := loopInfo{} + copy(info.fileName[:], []byte(backingFile)) + if param.Readonly { + info.flags |= loFlagsReadonly + } + if param.Autoclear { + info.flags |= loFlagsAutoclear + } + if param.Direct { + info.flags |= loFlagsAutoclear + } + if _, _, err := ioctl(loopFile.Fd(), ioctlSetStatus64, uintptr(unsafe.Pointer(&info))); err != nil { + // Retry w/o direct IO flag in case kernel does not support it. The downside is that + // it will suffer from double cache problem. + info.flags &= ^(uint32(loFlagsDirectIO)) + if _, _, err := ioctl(loopFile.Fd(), ioctlSetStatus64, uintptr(unsafe.Pointer(&info))); err != nil { + ioctl(loopFile.Fd(), ioctlClrFd, 0) + return nil, errors.Errorf("cannot set loop info:%v", err) + } + } + + return loopFile, nil +} + +// setupLoop looks for (and possibly creates) a free loop device, and +// then attaches backingFile to it. +// +// When autoclear is true, caller should take care to close it when +// done with the loop device. The loop device file handle keeps +// loFlagsAutoclear in effect and we rely on it to clean up the loop +// device. If caller closes the file handle after mounting the device, +// kernel will clear the loop device after it is umounted. Otherwise +// the loop device is cleared when the file handle is closed. +// +// When autoclear is false, caller should be responsible to remove +// the loop device when done with it. +// +// Upon success, the file handle to the loop device is returned. +func setupLoop(backingFile string, param LoopParams) (*os.File, error) { + var loopDev string + + for retry := 1; retry < 200; retry++ { + num, err := getFreeLoopDev() + if err != nil { + return nil, err + } + + loopDev = fmt.Sprintf(loopDevFormat, num) + loopFile, err := setupLoopDev(backingFile, loopDev, param) + if err != nil { + // Per util-linux/sys-utils/losetup.c:create_loop(), + // free loop device can race and we end up failing + // with EBUSY when trying to set it up. + if strings.Contains(err.Error(), ebusyString) { + // Fallback a bit to avoid live lock + time.Sleep(time.Millisecond * time.Duration(rand.Intn(retry*10))) + continue + } + return nil, err + } + return loopFile, nil + } + + return nil, errors.New("Timeout creating new loopback device") +} + +func removeLoop(loopdev string) error { + dev, err := os.Open(loopdev) + if err != nil { + return err + } + _, _, err = ioctl(dev.Fd(), ioctlClrFd, 0) + dev.Close() + return err +} + +// Attach a specified backing file to a loop device +func AttachLoopDevice(backingFile string) (string, error) { + dev, err := setupLoop(backingFile, LoopParams{}) + if err != nil { + return "", err + } + dev.Close() + + return dev.Name(), nil +} + +// Detach a loop device +func DetachLoopDevice(devices ...string) error { + for _, dev := range devices { + if err := removeLoop(dev); err != nil { + return err + } + } + + return nil +} diff --git a/mount/losetup_linux_test.go b/mount/losetup_linux_test.go new file mode 100644 index 000000000..6ed1e0eb8 --- /dev/null +++ b/mount/losetup_linux_test.go @@ -0,0 +1,117 @@ +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package mount + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/containerd/continuity/testutil" +) + +func TestSetupLoop(t *testing.T) { + testutil.RequiresRoot(t) + const randomdata = "randomdata" + + /* Non-existing loop */ + backingFile := "setup-loop-test-no-such-file" + _, err := setupLoop(backingFile, LoopParams{}) + if err == nil { + t.Fatalf("setupLoop with non-existing file should fail") + } + + f, err := ioutil.TempFile("", "losetup") + if err != nil { + t.Fatal(err) + } + if err = f.Truncate(512); err != nil { + t.Fatal(err) + } + backingFile = f.Name() + f.Close() + defer func() { + if err := os.Remove(backingFile); err != nil { + t.Fatal(err) + } + }() + + /* RO loop */ + f, err = setupLoop(backingFile, LoopParams{Readonly: true, Autoclear: true}) + if err != nil { + t.Fatal(err) + } + ff, err := os.OpenFile(f.Name(), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err = ff.Write([]byte(randomdata)); err == nil { + t.Fatalf("writing to readonly loop device should fail") + } + if err = ff.Close(); err != nil { + t.Fatal(err) + } + if err = f.Close(); err != nil { + t.Fatal(err) + } + + /* RW loop */ + f, err = setupLoop(backingFile, LoopParams{Autoclear: true}) + if err != nil { + t.Fatal(err) + } + ff, err = os.OpenFile(f.Name(), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err = ff.Write([]byte(randomdata)); err != nil { + t.Fatal(err) + } + if err = ff.Close(); err != nil { + t.Fatal(err) + } + if err = f.Close(); err != nil { + t.Fatal(err) + } +} + +func TestAttachDetachLoopDevice(t *testing.T) { + testutil.RequiresRoot(t) + f, err := ioutil.TempFile("", "losetup") + if err != nil { + t.Fatal(err) + } + if err = f.Truncate(512); err != nil { + t.Fatal(err) + } + f.Close() + defer func() { + if err := os.Remove(f.Name()); err != nil { + t.Fatal(err) + } + }() + + dev, err := AttachLoopDevice(f.Name()) + if err != nil { + t.Fatal(err) + } + if err = DetachLoopDevice(dev); err != nil { + t.Fatal(err) + } +} diff --git a/mount/mount_linux.go b/mount/mount_linux.go index 6d28f0a97..b0e24f8f0 100644 --- a/mount/mount_linux.go +++ b/mount/mount_linux.go @@ -42,7 +42,7 @@ func init() { // // If m.Type starts with "fuse." or "fuse3.", "mount.fuse" or "mount.fuse3" // helper binary is called. -func (m *Mount) Mount(target string) error { +func (m *Mount) Mount(target string) (err error) { for _, helperBinary := range allowedHelperBinaries { // helperBinary = "mount.fuse", typePrefix = "fuse." typePrefix := strings.TrimPrefix(helperBinary, "mount.") + "." @@ -62,7 +62,7 @@ func (m *Mount) Mount(target string) error { chdir, options = compactLowerdirOption(options) } - flags, data := parseMountOptions(options) + flags, data, losetup := parseMountOptions(options) if len(data) > pagesize { return errors.Errorf("mount options is too long") } @@ -77,7 +77,19 @@ func (m *Mount) Mount(target string) error { if flags&unix.MS_REMOUNT == 0 || data != "" { // Initial call applying all non-propagation flags for mount // or remount with changed data - if err := mountAt(chdir, m.Source, target, m.Type, uintptr(oflags), data); err != nil { + source := m.Source + if losetup { + devFile, err := setupLoop(m.Source, LoopParams{ + Readonly: oflags&unix.MS_RDONLY == unix.MS_RDONLY, + Autoclear: true}) + if err != nil { + return err + } + defer devFile.Close() + // Mount the loop device instead + source = devFile.Name() + } + if err := mountAt(chdir, source, target, m.Type, uintptr(oflags), data); err != nil { return err } } @@ -186,11 +198,13 @@ func UnmountAll(mount string, flags int) error { // parseMountOptions takes fstab style mount options and parses them for // use with a standard mount() syscall -func parseMountOptions(options []string) (int, string) { +func parseMountOptions(options []string) (int, string, bool) { var ( - flag int - data []string + flag int + losetup bool + data []string ) + loopOpt := "loop" flags := map[string]struct { clear bool flag int @@ -231,11 +245,13 @@ func parseMountOptions(options []string) (int, string) { } else { flag |= f.flag } + } else if o == loopOpt { + losetup = true } else { data = append(data, o) } } - return flag, strings.Join(data, ",") + return flag, strings.Join(data, ","), losetup } // compactLowerdirOption updates overlay lowdir option and returns the common