From ef98c71985cf9c67b1cea312a9b044fc6042e5f3 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 9 Jan 2024 10:43:13 -0500 Subject: [PATCH] Implement fsverity functionality Implement calls to the fsverity kernel module, allowing containerd to enable fsverity on blob data in the content store. This causes fsverity to veirfy the integrity of blob data when the blob is read. Signed-off-by: James Jenkins --- pkg/fsverity/fsverity_linux.go | 130 +++++++++++++++ pkg/fsverity/fsverity_other.go | 33 ++++ pkg/fsverity/fsverity_test.go | 236 ++++++++++++++++++++++++++++ plugins/content/local/store.go | 13 +- plugins/content/local/store_test.go | 13 ++ plugins/content/local/writer.go | 9 ++ 6 files changed, 430 insertions(+), 4 deletions(-) create mode 100644 pkg/fsverity/fsverity_linux.go create mode 100644 pkg/fsverity/fsverity_other.go create mode 100644 pkg/fsverity/fsverity_test.go diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go new file mode 100644 index 000000000..f19f0e9ce --- /dev/null +++ b/pkg/fsverity/fsverity_linux.go @@ -0,0 +1,130 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fsverity + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "unsafe" + + "github.com/containerd/containerd/v2/pkg/kernelversion" + "golang.org/x/sys/unix" +) + +type fsverityEnableArg struct { + version uint32 + hashAlgorithm uint32 + blockSize uint32 + saltSize uint32 + saltPtr uint64 + sigSize uint32 + reserved1 uint32 + sigPtr uint64 + reserved2 [11]uint64 +} + +const ( + defaultBlockSize int = 4096 + maxDigestSize uint16 = 64 +) + +func IsSupported(rootPath string) (bool, error) { + minKernelVersion := kernelversion.KernelVersion{Kernel: 5, Major: 4} + s, err := kernelversion.GreaterEqualThan(minKernelVersion) + if err != nil { + return s, err + } + + integrityDir, err := os.MkdirTemp(rootPath, ".fsverity-check-*") + if err != nil { + return false, err + } + defer os.RemoveAll(integrityDir) + + digestPath := filepath.Join(integrityDir, "supported") + digestFile, err := os.Create(digestPath) + if err != nil { + return false, err + } + + digestFile.Close() + + eerr := Enable(digestPath) + if eerr != nil { + return false, eerr + } + + return true, nil +} + +func IsEnabled(path string) (bool, error) { + f, err := os.Open(path) + if err != nil { + return false, err + } + defer f.Close() + + var attr int + + _, _, flagErr := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_GETFLAGS), uintptr(unsafe.Pointer(&attr))) + if flagErr != 0 { + return false, fmt.Errorf("error getting inode flags: %w", flagErr) + } + + if attr&unix.FS_VERITY_FL == unix.FS_VERITY_FL { + return true, nil + } + + return false, nil +} + +func Enable(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + var args = &fsverityEnableArg{} + args.version = 1 + args.hashAlgorithm = 1 + + // fsverity block size should be the minimum between the page size + // and the file system block size + // If neither value is retrieved successfully, set fsverity block size to the default value + blockSize := unix.Getpagesize() + + s := unix.Stat_t{} + serr := unix.Stat(path, &s) + if serr == nil && int(s.Blksize) < blockSize { + blockSize = int(s.Blksize) + } + + if blockSize <= 0 { + blockSize = defaultBlockSize + } + + args.blockSize = uint32(blockSize) + + _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args))) + if errno != 0 { + return fmt.Errorf("enable fsverity failed: %w", errno) + } + + return nil +} diff --git a/pkg/fsverity/fsverity_other.go b/pkg/fsverity/fsverity_other.go new file mode 100644 index 000000000..f50789ada --- /dev/null +++ b/pkg/fsverity/fsverity_other.go @@ -0,0 +1,33 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fsverity + +import "fmt" + +func IsSupported(rootPath string) (bool, error) { + return false, fmt.Errorf("fsverity is only supported on Linux systems") +} + +func IsEnabled(path string) (bool, error) { + return false, fmt.Errorf("fsverity is only supported on Linux systems") +} + +func Enable(_ string) error { + return fmt.Errorf("fsverity is only supported on Linux systems") +} diff --git a/pkg/fsverity/fsverity_test.go b/pkg/fsverity/fsverity_test.go new file mode 100644 index 000000000..1bd122ba5 --- /dev/null +++ b/pkg/fsverity/fsverity_test.go @@ -0,0 +1,236 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fsverity + +import ( + "bufio" + "encoding/binary" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/containerd/containerd/v2/pkg/testutil" +) + +type superblockROFeatures struct { + _ [100]byte + Features uint32 +} + +func TestEnable(t *testing.T) { + + testutil.RequiresRoot(t) + + rootDir := filepath.Join(t.TempDir(), "content") + err := os.Mkdir(rootDir, 0755) + if err != nil { + t.Errorf("could not create temporary directory: %s", err.Error()) + } + + device, err := resolveDevicePath(rootDir) + if err != nil { + t.Skipf("invalid device: %s", err.Error()) + } + + var expected bool + enabled, err := ext4IsVerity(device) + if !enabled || err != nil { + t.Logf("fsverity not enabled on ext4 file system: %s", err.Error()) + expected = false + } else { + t.Logf("fsverity enabled on ext4 file system") + expected = true + } + + verityFile := filepath.Join(rootDir, "fsverityFile") + f, err := os.Create(verityFile) + if err != nil { + t.Errorf("could not create fsverity test file: %s", err.Error()) + } + + err = f.Close() + if err != nil { + t.Errorf("error closing fsverity test file: %s", err.Error()) + } + + defer func() { + err := os.Remove(verityFile) + if err != nil { + t.Logf("error removing fsverity test file: %s", err.Error()) + } + }() + + err = Enable(verityFile) + if err != nil && expected { + t.Errorf("fsverity Enable failed: %s", err.Error()) + } + if err == nil && !expected { + t.Errorf("fsverity Enable succeeded, expected Enable to fail") + } +} + +func TestIsEnabled(t *testing.T) { + + testDir := filepath.Join(t.TempDir(), "content") + err := os.Mkdir(testDir, 0755) + if err != nil { + t.Errorf("could not create temporary directory: %s", err.Error()) + } + + if supported, err := IsSupported(testDir); !supported || err != nil { + t.Skipf("fsverity is not supported") + } + + verityFile := filepath.Join(testDir, "fsverityFile") + f, err := os.Create(verityFile) + if err != nil { + t.Errorf("could not create fsverity test file: %s", err.Error()) + } + + err = f.Close() + if err != nil { + t.Errorf("error closing fsverity test file: %s", err.Error()) + } + + defer func() { + err := os.Remove(verityFile) + if err != nil { + t.Logf("error removing fsverity test file: %s", err.Error()) + } + }() + + err = Enable(verityFile) + if err != nil { + t.Errorf("fsverity Enable failed: %s", err.Error()) + } + + if enabled, err := IsEnabled(verityFile); !enabled || err != nil { + t.Errorf("expected fsverity to be enabled on file, received enabled: %t; error: %s", enabled, err.Error()) + } +} + +func resolveDevicePath(path string) (_ string, e error) { + var devicePath string + + s, err := os.Stat(path) + if err != nil { + return devicePath, err + } + + sys := s.Sys() + stat, ok := sys.(*syscall.Stat_t) + if !ok { + return devicePath, fmt.Errorf("type assert to syscall.Stat_t failed") + } + + // resolve to device path + maj := (stat.Dev >> 8) & 0xff + min := stat.Dev & 0xff + + m, err := os.Open("/proc/self/mountinfo") + if err != nil { + return devicePath, err + } + + defer func() { + err := m.Close() + if err != nil { + e = fmt.Errorf("could not close mountinfo: %v", err) + } + }() + + // scan for major:minor id and get the device path + scanner := bufio.NewScanner(m) + + var entry string + sub := fmt.Sprintf("%d:%d", maj, min) + for scanner.Scan() { + if strings.Contains(scanner.Text(), sub) { + entry = scanner.Text() + break + } + } + if entry == "" { + return devicePath, fmt.Errorf("device mount not found for device id %s", sub) + } + + entryReader := strings.NewReader(entry) + extScan := bufio.NewScanner(entryReader) + extScan.Split(bufio.ScanWords) + + var word string + for (word != "-") && extScan.Scan() { + word = extScan.Text() + } + + if !extScan.Scan() { + return devicePath, fmt.Errorf("scanning mounts failed: %w", extScan.Err()) + } + fs := extScan.Text() + + if fs != "ext4" { + return devicePath, fmt.Errorf("not an ext4 file system, skipping device") + } + + if !extScan.Scan() { + return devicePath, fmt.Errorf("scanning mounts failed: %w", extScan.Err()) + } + devicePath = extScan.Text() + + return devicePath, nil +} + +func ext4IsVerity(fpath string) (bool, error) { + b := superblockROFeatures{} + + r, err := os.Open(fpath) + if err != nil { + return false, err + } + + defer func() { + err := r.Close() + if err != nil { + fmt.Printf("failed to close %s: %s\n", fpath, err.Error()) + } + }() + + // seek to superblock + _, err = r.Seek(1024, 0) + if err != nil { + return false, err + } + + err = binary.Read(r, binary.LittleEndian, &b) + if err != nil { + return false, err + } + + // extract fsverity flag + var verityMask uint32 = 0x8000 + res := verityMask & b.Features + if res > 0 { + return true, nil + } + + return false, fmt.Errorf("fsverity not enabled on ext4 file system %s", fpath) +} diff --git a/plugins/content/local/store.go b/plugins/content/local/store.go index 4b06351a0..a15265864 100644 --- a/plugins/content/local/store.go +++ b/plugins/content/local/store.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/pkg/filters" + "github.com/containerd/containerd/v2/pkg/fsverity" "github.com/containerd/errdefs" "github.com/containerd/log" @@ -62,8 +63,9 @@ type LabelStore interface { // Store can generally support multi-reader, single-writer ingest of data, // including resumable ingest. type store struct { - root string - ls LabelStore + root string + ls LabelStore + integritySupported bool } // NewStore returns a local content store @@ -81,9 +83,12 @@ func NewLabeledStore(root string, ls LabelStore) (content.Store, error) { return nil, err } + supported, _ := fsverity.IsSupported(root) + return &store{ - root: root, - ls: ls, + root: root, + ls: ls, + integritySupported: supported, }, nil } diff --git a/plugins/content/local/store_test.go b/plugins/content/local/store_test.go index 03e1bbff9..9c1795c46 100644 --- a/plugins/content/local/store_test.go +++ b/plugins/content/local/store_test.go @@ -35,6 +35,7 @@ import ( "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/content/testsuite" "github.com/containerd/containerd/v2/internal/randutil" + "github.com/containerd/containerd/v2/pkg/fsverity" "github.com/containerd/containerd/v2/pkg/testutil" "github.com/containerd/errdefs" @@ -193,6 +194,18 @@ func TestContentWriter(t *testing.T) { t.Fatal("mismatched data written to disk") } + // ensure fsverity is enabled on blob if fsverity is supported + ok, err := fsverity.IsSupported(tmpdir) + if !ok || err != nil { + t.Log("fsverity not supported, skipping fsverity check") + return + } + + ok, err = fsverity.IsEnabled(path) + if !ok || err != nil { + t.Fatal(err) + } + } func TestWalkBlobs(t *testing.T) { diff --git a/plugins/content/local/writer.go b/plugins/content/local/writer.go index 26cef0aae..c20d0cf2c 100644 --- a/plugins/content/local/writer.go +++ b/plugins/content/local/writer.go @@ -27,6 +27,7 @@ import ( "time" "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/pkg/fsverity" "github.com/containerd/errdefs" "github.com/containerd/log" "github.com/opencontainers/go-digest" @@ -137,6 +138,14 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, return err } + // Enable content blob integrity verification if supported + + if w.s.integritySupported { + if err := fsverity.Enable(target); err != nil { + log.G(ctx).Warnf("failed to enable integrity for blob %v: %s", target, err.Error()) + } + } + // Ingest has now been made available in the content store, attempt to complete // setting metadata but errors should only be logged and not returned since // the content store cannot be cleanly rolled back.