Merge pull request #10007 from Jenkins-J/fsverity-content-verification

Fsverity content verification

pkg/fsverity/fsverity_linux.go

@@ -0,0 +1,130 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fsverity
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+
+	"github.com/containerd/containerd/v2/pkg/kernelversion"
+	"golang.org/x/sys/unix"
+)
+
+type fsverityEnableArg struct {
+	version       uint32
+	hashAlgorithm uint32
+	blockSize     uint32
+	saltSize      uint32
+	saltPtr       uint64
+	sigSize       uint32
+	reserved1     uint32
+	sigPtr        uint64
+	reserved2     [11]uint64
+}
+
+const (
+	defaultBlockSize int    = 4096
+	maxDigestSize    uint16 = 64
+)
+
+func IsSupported(rootPath string) (bool, error) {
+	minKernelVersion := kernelversion.KernelVersion{Kernel: 5, Major: 4}
+	s, err := kernelversion.GreaterEqualThan(minKernelVersion)
+	if err != nil {
+		return s, err
+	}
+
+	integrityDir, err := os.MkdirTemp(rootPath, ".fsverity-check-*")
+	if err != nil {
+		return false, err
+	}
+	defer os.RemoveAll(integrityDir)
+
+	digestPath := filepath.Join(integrityDir, "supported")
+	digestFile, err := os.Create(digestPath)
+	if err != nil {
+		return false, err
+	}
+
+	digestFile.Close()
+
+	eerr := Enable(digestPath)
+	if eerr != nil {
+		return false, eerr
+	}
+
+	return true, nil
+}
+
+func IsEnabled(path string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	var attr int
+
+	_, _, flagErr := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_GETFLAGS), uintptr(unsafe.Pointer(&attr)))
+	if flagErr != 0 {
+		return false, fmt.Errorf("error getting inode flags: %w", flagErr)
+	}
+
+	if attr&unix.FS_VERITY_FL == unix.FS_VERITY_FL {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func Enable(path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	var args = &fsverityEnableArg{}
+	args.version = 1
+	args.hashAlgorithm = 1
+
+	// fsverity block size should be the minimum between the page size
+	// and the file system block size
+	// If neither value is retrieved successfully, set fsverity block size to the default value
+	blockSize := unix.Getpagesize()
+
+	s := unix.Stat_t{}
+	serr := unix.Stat(path, &s)
+	if serr == nil && int(s.Blksize) < blockSize {
+		blockSize = int(s.Blksize)
+	}
+
+	if blockSize <= 0 {
+		blockSize = defaultBlockSize
+	}
+
+	args.blockSize = uint32(blockSize)
+
+	_, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args)))
+	if errno != 0 {
+		return fmt.Errorf("enable fsverity failed: %w", errno)
+	}
+
+	return nil
+}

pkg/fsverity/fsverity_other.go

@@ -0,0 +1,33 @@
+//go:build !linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fsverity
+
+import "fmt"
+
+func IsSupported(rootPath string) (bool, error) {
+	return false, fmt.Errorf("fsverity is only supported on Linux systems")
+}
+
+func IsEnabled(path string) (bool, error) {
+	return false, fmt.Errorf("fsverity is only supported on Linux systems")
+}
+
+func Enable(_ string) error {
+	return fmt.Errorf("fsverity is only supported on Linux systems")
+}

pkg/fsverity/fsverity_test.go

@@ -0,0 +1,236 @@
+//go:build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fsverity
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/containerd/containerd/v2/pkg/testutil"
+)
+
+type superblockROFeatures struct {
+	_        [100]byte
+	Features uint32
+}
+
+func TestEnable(t *testing.T) {
+
+	testutil.RequiresRoot(t)
+
+	rootDir := filepath.Join(t.TempDir(), "content")
+	err := os.Mkdir(rootDir, 0755)
+	if err != nil {
+		t.Errorf("could not create temporary directory: %s", err.Error())
+	}
+
+	device, err := resolveDevicePath(rootDir)
+	if err != nil {
+		t.Skipf("invalid device: %s", err.Error())
+	}
+
+	var expected bool
+	enabled, err := ext4IsVerity(device)
+	if !enabled || err != nil {
+		t.Logf("fsverity not enabled on ext4 file system: %s", err.Error())
+		expected = false
+	} else {
+		t.Logf("fsverity enabled on ext4 file system")
+		expected = true
+	}
+
+	verityFile := filepath.Join(rootDir, "fsverityFile")
+	f, err := os.Create(verityFile)
+	if err != nil {
+		t.Errorf("could not create fsverity test file: %s", err.Error())
+	}
+
+	err = f.Close()
+	if err != nil {
+		t.Errorf("error closing fsverity test file: %s", err.Error())
+	}
+
+	defer func() {
+		err := os.Remove(verityFile)
+		if err != nil {
+			t.Logf("error removing fsverity test file: %s", err.Error())
+		}
+	}()
+
+	err = Enable(verityFile)
+	if err != nil && expected {
+		t.Errorf("fsverity Enable failed: %s", err.Error())
+	}
+	if err == nil && !expected {
+		t.Errorf("fsverity Enable succeeded, expected Enable to fail")
+	}
+}
+
+func TestIsEnabled(t *testing.T) {
+
+	testDir := filepath.Join(t.TempDir(), "content")
+	err := os.Mkdir(testDir, 0755)
+	if err != nil {
+		t.Errorf("could not create temporary directory: %s", err.Error())
+	}
+
+	if supported, err := IsSupported(testDir); !supported || err != nil {
+		t.Skipf("fsverity is not supported")
+	}
+
+	verityFile := filepath.Join(testDir, "fsverityFile")
+	f, err := os.Create(verityFile)
+	if err != nil {
+		t.Errorf("could not create fsverity test file: %s", err.Error())
+	}
+
+	err = f.Close()
+	if err != nil {
+		t.Errorf("error closing fsverity test file: %s", err.Error())
+	}
+
+	defer func() {
+		err := os.Remove(verityFile)
+		if err != nil {
+			t.Logf("error removing fsverity test file: %s", err.Error())
+		}
+	}()
+
+	err = Enable(verityFile)
+	if err != nil {
+		t.Errorf("fsverity Enable failed: %s", err.Error())
+	}
+
+	if enabled, err := IsEnabled(verityFile); !enabled || err != nil {
+		t.Errorf("expected fsverity to be enabled on file, received enabled: %t; error: %s", enabled, err.Error())
+	}
+}
+
+func resolveDevicePath(path string) (_ string, e error) {
+	var devicePath string
+
+	s, err := os.Stat(path)
+	if err != nil {
+		return devicePath, err
+	}
+
+	sys := s.Sys()
+	stat, ok := sys.(*syscall.Stat_t)
+	if !ok {
+		return devicePath, fmt.Errorf("type assert to syscall.Stat_t failed")
+	}
+
+	// resolve to device path
+	maj := (stat.Dev >> 8) & 0xff
+	min := stat.Dev & 0xff
+
+	m, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return devicePath, err
+	}
+
+	defer func() {
+		err := m.Close()
+		if err != nil {
+			e = fmt.Errorf("could not close mountinfo: %v", err)
+		}
+	}()
+
+	// scan for major:minor id and get the device path
+	scanner := bufio.NewScanner(m)
+
+	var entry string
+	sub := fmt.Sprintf("%d:%d", maj, min)
+	for scanner.Scan() {
+		if strings.Contains(scanner.Text(), sub) {
+			entry = scanner.Text()
+			break
+		}
+	}
+	if entry == "" {
+		return devicePath, fmt.Errorf("device mount not found for device id %s", sub)
+	}
+
+	entryReader := strings.NewReader(entry)
+	extScan := bufio.NewScanner(entryReader)
+	extScan.Split(bufio.ScanWords)
+
+	var word string
+	for (word != "-") && extScan.Scan() {
+		word = extScan.Text()
+	}
+
+	if !extScan.Scan() {
+		return devicePath, fmt.Errorf("scanning mounts failed: %w", extScan.Err())
+	}
+	fs := extScan.Text()
+
+	if fs != "ext4" {
+		return devicePath, fmt.Errorf("not an ext4 file system, skipping device")
+	}
+
+	if !extScan.Scan() {
+		return devicePath, fmt.Errorf("scanning mounts failed: %w", extScan.Err())
+	}
+	devicePath = extScan.Text()
+
+	return devicePath, nil
+}
+
+func ext4IsVerity(fpath string) (bool, error) {
+	b := superblockROFeatures{}
+
+	r, err := os.Open(fpath)
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		err := r.Close()
+		if err != nil {
+			fmt.Printf("failed to close %s: %s\n", fpath, err.Error())
+		}
+	}()
+
+	// seek to superblock
+	_, err = r.Seek(1024, 0)
+	if err != nil {
+		return false, err
+	}
+
+	err = binary.Read(r, binary.LittleEndian, &b)
+	if err != nil {
+		return false, err
+	}
+
+	// extract fsverity flag
+	var verityMask uint32 = 0x8000
+	res := verityMask & b.Features
+	if res > 0 {
+		return true, nil
+	}
+
+	return false, fmt.Errorf("fsverity not enabled on ext4 file system %s", fpath)
+}

plugins/content/local/store.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/pkg/filters"
+	"github.com/containerd/containerd/v2/pkg/fsverity"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 
@@ -62,8 +63,9 @@ type LabelStore interface {
 // Store can generally support multi-reader, single-writer ingest of data,
 // including resumable ingest.
 type store struct {
-	root string
+	root               string
-	ls   LabelStore
+	ls                 LabelStore
+	integritySupported bool
 }
 
 // NewStore returns a local content store
@@ -81,9 +83,12 @@ func NewLabeledStore(root string, ls LabelStore) (content.Store, error) {
 		return nil, err
 	}
 
+	supported, _ := fsverity.IsSupported(root)
+
 	return &store{
-		root: root,
+		root:               root,
-		ls:   ls,
+		ls:                 ls,
+		integritySupported: supported,
 	}, nil
 }
 

plugins/content/local/store_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/content/testsuite"
 	"github.com/containerd/containerd/v2/internal/randutil"
+	"github.com/containerd/containerd/v2/pkg/fsverity"
 	"github.com/containerd/containerd/v2/pkg/testutil"
 	"github.com/containerd/errdefs"
 
@@ -193,6 +194,18 @@ func TestContentWriter(t *testing.T) {
 		t.Fatal("mismatched data written to disk")
 	}
 
+	// ensure fsverity is enabled on blob if fsverity is supported
+	ok, err := fsverity.IsSupported(tmpdir)
+	if !ok || err != nil {
+		t.Log("fsverity not supported, skipping fsverity check")
+		return
+	}
+
+	ok, err = fsverity.IsEnabled(path)
+	if !ok || err != nil {
+		t.Fatal(err)
+	}
+
 }
 
 func TestWalkBlobs(t *testing.T) {

plugins/content/local/writer.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/pkg/fsverity"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	"github.com/opencontainers/go-digest"
@@ -137,6 +138,14 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest,
 		return err
 	}
 
+	// Enable content blob integrity verification if supported
+
+	if w.s.integritySupported {
+		if err := fsverity.Enable(target); err != nil {
+			log.G(ctx).Warnf("failed to enable integrity for blob %v: %s", target, err.Error())
+		}
+	}
+
 	// Ingest has now been made available in the content store, attempt to complete
 	// setting metadata but errors should only be logged and not returned since
 	// the content store cannot be cleanly rolled back.