cri: implement RuntimeConfig rpc

The rpc only reports one field, i.e. the cgroup driver, to kubelet.
Containerd determines the effective cgroup driver by looking at all
runtime handlers, starting from the default runtime handler (the rest in
alphabetical order), and returning the cgroup driver setting of the
first runtime handler that supports one. If no runtime handler supports
cgroup driver (i.e. has a config option for it) containerd falls back to
auto-detection, returning systemd if systemd is running and cgroupfs
otherwise.

This patch implements the CRI server side of Kubernetes KEP-4033:
https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4033-group-driver-detection-over-cri

Signed-off-by: Markus Lehtonen <markus.lehtonen@intel.com>

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/cpu.go

@@ -0,0 +1,87 @@
+package fs2
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func isCpuSet(r *configs.Resources) bool {
+	return r.CpuWeight != 0 || r.CpuQuota != 0 || r.CpuPeriod != 0
+}
+
+func setCpu(dirPath string, r *configs.Resources) error {
+	if !isCpuSet(r) {
+		return nil
+	}
+
+	// NOTE: .CpuShares is not used here. Conversion is the caller's responsibility.
+	if r.CpuWeight != 0 {
+		if err := cgroups.WriteFile(dirPath, "cpu.weight", strconv.FormatUint(r.CpuWeight, 10)); err != nil {
+			return err
+		}
+	}
+
+	if r.CpuQuota != 0 || r.CpuPeriod != 0 {
+		str := "max"
+		if r.CpuQuota > 0 {
+			str = strconv.FormatInt(r.CpuQuota, 10)
+		}
+		period := r.CpuPeriod
+		if period == 0 {
+			// This default value is documented in
+			// https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
+			period = 100000
+		}
+		str += " " + strconv.FormatUint(period, 10)
+		if err := cgroups.WriteFile(dirPath, "cpu.max", str); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func statCpu(dirPath string, stats *cgroups.Stats) error {
+	const file = "cpu.stat"
+	f, err := cgroups.OpenFile(dirPath, file, os.O_RDONLY)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		t, v, err := fscommon.ParseKeyValue(sc.Text())
+		if err != nil {
+			return &parseError{Path: dirPath, File: file, Err: err}
+		}
+		switch t {
+		case "usage_usec":
+			stats.CpuStats.CpuUsage.TotalUsage = v * 1000
+
+		case "user_usec":
+			stats.CpuStats.CpuUsage.UsageInUsermode = v * 1000
+
+		case "system_usec":
+			stats.CpuStats.CpuUsage.UsageInKernelmode = v * 1000
+
+		case "nr_periods":
+			stats.CpuStats.ThrottlingData.Periods = v
+
+		case "nr_throttled":
+			stats.CpuStats.ThrottlingData.ThrottledPeriods = v
+
+		case "throttled_usec":
+			stats.CpuStats.ThrottlingData.ThrottledTime = v * 1000
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return &parseError{Path: dirPath, File: file, Err: err}
+	}
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/cpuset.go

@@ -0,0 +1,28 @@
+package fs2
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func isCpusetSet(r *configs.Resources) bool {
+	return r.CpusetCpus != "" || r.CpusetMems != ""
+}
+
+func setCpuset(dirPath string, r *configs.Resources) error {
+	if !isCpusetSet(r) {
+		return nil
+	}
+
+	if r.CpusetCpus != "" {
+		if err := cgroups.WriteFile(dirPath, "cpuset.cpus", r.CpusetCpus); err != nil {
+			return err
+		}
+	}
+	if r.CpusetMems != "" {
+		if err := cgroups.WriteFile(dirPath, "cpuset.mems", r.CpusetMems); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/create.go

@@ -0,0 +1,152 @@
+package fs2
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func supportedControllers() (string, error) {
+	return cgroups.ReadFile(UnifiedMountpoint, "/cgroup.controllers")
+}
+
+// needAnyControllers returns whether we enable some supported controllers or not,
+// based on (1) controllers available and (2) resources that are being set.
+// We don't check "pseudo" controllers such as
+// "freezer" and "devices".
+func needAnyControllers(r *configs.Resources) (bool, error) {
+	if r == nil {
+		return false, nil
+	}
+
+	// list of all available controllers
+	content, err := supportedControllers()
+	if err != nil {
+		return false, err
+	}
+	avail := make(map[string]struct{})
+	for _, ctr := range strings.Fields(content) {
+		avail[ctr] = struct{}{}
+	}
+
+	// check whether the controller if available or not
+	have := func(controller string) bool {
+		_, ok := avail[controller]
+		return ok
+	}
+
+	if isPidsSet(r) && have("pids") {
+		return true, nil
+	}
+	if isMemorySet(r) && have("memory") {
+		return true, nil
+	}
+	if isIoSet(r) && have("io") {
+		return true, nil
+	}
+	if isCpuSet(r) && have("cpu") {
+		return true, nil
+	}
+	if isCpusetSet(r) && have("cpuset") {
+		return true, nil
+	}
+	if isHugeTlbSet(r) && have("hugetlb") {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// containsDomainController returns whether the current config contains domain controller or not.
+// Refer to: http://man7.org/linux/man-pages/man7/cgroups.7.html
+// As at Linux 4.19, the following controllers are threaded: cpu, perf_event, and pids.
+func containsDomainController(r *configs.Resources) bool {
+	return isMemorySet(r) || isIoSet(r) || isCpuSet(r) || isHugeTlbSet(r)
+}
+
+// CreateCgroupPath creates cgroupv2 path, enabling all the supported controllers.
+func CreateCgroupPath(path string, c *configs.Cgroup) (Err error) {
+	if !strings.HasPrefix(path, UnifiedMountpoint) {
+		return fmt.Errorf("invalid cgroup path %s", path)
+	}
+
+	content, err := supportedControllers()
+	if err != nil {
+		return err
+	}
+
+	const (
+		cgTypeFile  = "cgroup.type"
+		cgStCtlFile = "cgroup.subtree_control"
+	)
+	ctrs := strings.Fields(content)
+	res := "+" + strings.Join(ctrs, " +")
+
+	elements := strings.Split(path, "/")
+	elements = elements[3:]
+	current := "/sys/fs"
+	for i, e := range elements {
+		current = filepath.Join(current, e)
+		if i > 0 {
+			if err := os.Mkdir(current, 0o755); err != nil {
+				if !os.IsExist(err) {
+					return err
+				}
+			} else {
+				// If the directory was created, be sure it is not left around on errors.
+				current := current
+				defer func() {
+					if Err != nil {
+						os.Remove(current)
+					}
+				}()
+			}
+			cgType, _ := cgroups.ReadFile(current, cgTypeFile)
+			cgType = strings.TrimSpace(cgType)
+			switch cgType {
+			// If the cgroup is in an invalid mode (usually this means there's an internal
+			// process in the cgroup tree, because we created a cgroup under an
+			// already-populated-by-other-processes cgroup), then we have to error out if
+			// the user requested controllers which are not thread-aware. However, if all
+			// the controllers requested are thread-aware we can simply put the cgroup into
+			// threaded mode.
+			case "domain invalid":
+				if containsDomainController(c.Resources) {
+					return fmt.Errorf("cannot enter cgroupv2 %q with domain controllers -- it is in an invalid state", current)
+				} else {
+					// Not entirely correct (in theory we'd always want to be a domain --
+					// since that means we're a properly delegated cgroup subtree) but in
+					// this case there's not much we can do and it's better than giving an
+					// error.
+					_ = cgroups.WriteFile(current, cgTypeFile, "threaded")
+				}
+			// If the cgroup is in (threaded) or (domain threaded) mode, we can only use thread-aware controllers
+			// (and you cannot usually take a cgroup out of threaded mode).
+			case "domain threaded":
+				fallthrough
+			case "threaded":
+				if containsDomainController(c.Resources) {
+					return fmt.Errorf("cannot enter cgroupv2 %q with domain controllers -- it is in %s mode", current, cgType)
+				}
+			}
+		}
+		// enable all supported controllers
+		if i < len(elements)-1 {
+			if err := cgroups.WriteFile(current, cgStCtlFile, res); err != nil {
+				// try write one by one
+				allCtrs := strings.Split(res, " ")
+				for _, ctr := range allCtrs {
+					_ = cgroups.WriteFile(current, cgStCtlFile, ctr)
+				}
+			}
+			// Some controllers might not be enabled when rootless or containerized,
+			// but we don't catch the error here. (Caught in setXXX() functions.)
+		}
+	}
+
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/defaultpath.go

@@ -0,0 +1,99 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fs2
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/utils"
+)
+
+const UnifiedMountpoint = "/sys/fs/cgroup"
+
+func defaultDirPath(c *configs.Cgroup) (string, error) {
+	if (c.Name != "" || c.Parent != "") && c.Path != "" {
+		return "", fmt.Errorf("cgroup: either Path or Name and Parent should be used, got %+v", c)
+	}
+
+	return _defaultDirPath(UnifiedMountpoint, c.Path, c.Parent, c.Name)
+}
+
+func _defaultDirPath(root, cgPath, cgParent, cgName string) (string, error) {
+	if (cgName != "" || cgParent != "") && cgPath != "" {
+		return "", errors.New("cgroup: either Path or Name and Parent should be used")
+	}
+
+	// XXX: Do not remove CleanPath. Path safety is important! -- cyphar
+	innerPath := utils.CleanPath(cgPath)
+	if innerPath == "" {
+		cgParent := utils.CleanPath(cgParent)
+		cgName := utils.CleanPath(cgName)
+		innerPath = filepath.Join(cgParent, cgName)
+	}
+	if filepath.IsAbs(innerPath) {
+		return filepath.Join(root, innerPath), nil
+	}
+
+	ownCgroup, err := parseCgroupFile("/proc/self/cgroup")
+	if err != nil {
+		return "", err
+	}
+	// The current user scope most probably has tasks in it already,
+	// making it impossible to enable controllers for its sub-cgroup.
+	// A parent cgroup (with no tasks in it) is what we need.
+	ownCgroup = filepath.Dir(ownCgroup)
+
+	return filepath.Join(root, ownCgroup, innerPath), nil
+}
+
+// parseCgroupFile parses /proc/PID/cgroup file and return string
+func parseCgroupFile(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	return parseCgroupFromReader(f)
+}
+
+func parseCgroupFromReader(r io.Reader) (string, error) {
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		var (
+			text  = s.Text()
+			parts = strings.SplitN(text, ":", 3)
+		)
+		if len(parts) < 3 {
+			return "", fmt.Errorf("invalid cgroup entry: %q", text)
+		}
+		// text is like "0::/user.slice/user-1001.slice/session-1.scope"
+		if parts[0] == "0" && parts[1] == "" {
+			return parts[2], nil
+		}
+	}
+	if err := s.Err(); err != nil {
+		return "", err
+	}
+	return "", errors.New("cgroup path not found")
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/devices.go

@@ -0,0 +1,75 @@
+package fs2
+
+import (
+	"fmt"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups/ebpf"
+	"github.com/opencontainers/runc/libcontainer/cgroups/ebpf/devicefilter"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/devices"
+	"github.com/opencontainers/runc/libcontainer/userns"
+)
+
+func isRWM(perms devices.Permissions) bool {
+	var r, w, m bool
+	for _, perm := range perms {
+		switch perm {
+		case 'r':
+			r = true
+		case 'w':
+			w = true
+		case 'm':
+			m = true
+		}
+	}
+	return r && w && m
+}
+
+// This is similar to the logic applied in crun for handling errors from bpf(2)
+// <https://github.com/containers/crun/blob/0.17/src/libcrun/cgroup.c#L2438-L2470>.
+func canSkipEBPFError(r *configs.Resources) bool {
+	// If we're running in a user namespace we can ignore eBPF rules because we
+	// usually cannot use bpf(2), as well as rootless containers usually don't
+	// have the necessary privileges to mknod(2) device inodes or access
+	// host-level instances (though ideally we would be blocking device access
+	// for rootless containers anyway).
+	if userns.RunningInUserNS() {
+		return true
+	}
+
+	// We cannot ignore an eBPF load error if any rule if is a block rule or it
+	// doesn't permit all access modes.
+	//
+	// NOTE: This will sometimes trigger in cases where access modes are split
+	//       between different rules but to handle this correctly would require
+	//       using ".../libcontainer/cgroup/devices".Emulator.
+	for _, dev := range r.Devices {
+		if !dev.Allow || !isRWM(dev.Permissions) {
+			return false
+		}
+	}
+	return true
+}
+
+func setDevices(dirPath string, r *configs.Resources) error {
+	if r.SkipDevices {
+		return nil
+	}
+	insts, license, err := devicefilter.DeviceFilter(r.Devices)
+	if err != nil {
+		return err
+	}
+	dirFD, err := unix.Open(dirPath, unix.O_DIRECTORY|unix.O_RDONLY, 0o600)
+	if err != nil {
+		return fmt.Errorf("cannot get dir FD for %s", dirPath)
+	}
+	defer unix.Close(dirFD)
+	if _, err := ebpf.LoadAttachCgroupDeviceFilter(insts, license, dirFD); err != nil {
+		if !canSkipEBPFError(r) {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/freezer.go

@@ -0,0 +1,127 @@
+package fs2
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func setFreezer(dirPath string, state configs.FreezerState) error {
+	var stateStr string
+	switch state {
+	case configs.Undefined:
+		return nil
+	case configs.Frozen:
+		stateStr = "1"
+	case configs.Thawed:
+		stateStr = "0"
+	default:
+		return fmt.Errorf("invalid freezer state %q requested", state)
+	}
+
+	fd, err := cgroups.OpenFile(dirPath, "cgroup.freeze", unix.O_RDWR)
+	if err != nil {
+		// We can ignore this request as long as the user didn't ask us to
+		// freeze the container (since without the freezer cgroup, that's a
+		// no-op).
+		if state != configs.Frozen {
+			return nil
+		}
+		return fmt.Errorf("freezer not supported: %w", err)
+	}
+	defer fd.Close()
+
+	if _, err := fd.WriteString(stateStr); err != nil {
+		return err
+	}
+	// Confirm that the cgroup did actually change states.
+	if actualState, err := readFreezer(dirPath, fd); err != nil {
+		return err
+	} else if actualState != state {
+		return fmt.Errorf(`expected "cgroup.freeze" to be in state %q but was in %q`, state, actualState)
+	}
+	return nil
+}
+
+func getFreezer(dirPath string) (configs.FreezerState, error) {
+	fd, err := cgroups.OpenFile(dirPath, "cgroup.freeze", unix.O_RDONLY)
+	if err != nil {
+		// If the kernel is too old, then we just treat the freezer as being in
+		// an "undefined" state.
+		if os.IsNotExist(err) || errors.Is(err, unix.ENODEV) {
+			err = nil
+		}
+		return configs.Undefined, err
+	}
+	defer fd.Close()
+
+	return readFreezer(dirPath, fd)
+}
+
+func readFreezer(dirPath string, fd *os.File) (configs.FreezerState, error) {
+	if _, err := fd.Seek(0, 0); err != nil {
+		return configs.Undefined, err
+	}
+	state := make([]byte, 2)
+	if _, err := fd.Read(state); err != nil {
+		return configs.Undefined, err
+	}
+	switch string(state) {
+	case "0\n":
+		return configs.Thawed, nil
+	case "1\n":
+		return waitFrozen(dirPath)
+	default:
+		return configs.Undefined, fmt.Errorf(`unknown "cgroup.freeze" state: %q`, state)
+	}
+}
+
+// waitFrozen polls cgroup.events until it sees "frozen 1" in it.
+func waitFrozen(dirPath string) (configs.FreezerState, error) {
+	fd, err := cgroups.OpenFile(dirPath, "cgroup.events", unix.O_RDONLY)
+	if err != nil {
+		return configs.Undefined, err
+	}
+	defer fd.Close()
+
+	// XXX: Simple wait/read/retry is used here. An implementation
+	// based on poll(2) or inotify(7) is possible, but it makes the code
+	// much more complicated. Maybe address this later.
+	const (
+		// Perform maxIter with waitTime in between iterations.
+		waitTime = 10 * time.Millisecond
+		maxIter  = 1000
+	)
+	scanner := bufio.NewScanner(fd)
+	for i := 0; scanner.Scan(); {
+		if i == maxIter {
+			return configs.Undefined, fmt.Errorf("timeout of %s reached waiting for the cgroup to freeze", waitTime*maxIter)
+		}
+		line := scanner.Text()
+		val := strings.TrimPrefix(line, "frozen ")
+		if val != line { // got prefix
+			if val[0] == '1' {
+				return configs.Frozen, nil
+			}
+
+			i++
+			// wait, then re-read
+			time.Sleep(waitTime)
+			_, err := fd.Seek(0, 0)
+			if err != nil {
+				return configs.Undefined, err
+			}
+		}
+	}
+	// Should only reach here either on read error,
+	// or if the file does not contain "frozen " line.
+	return configs.Undefined, scanner.Err()
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/fs2.go

@@ -0,0 +1,259 @@
+package fs2
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type parseError = fscommon.ParseError
+
+type manager struct {
+	config *configs.Cgroup
+	// dirPath is like "/sys/fs/cgroup/user.slice/user-1001.slice/session-1.scope"
+	dirPath string
+	// controllers is content of "cgroup.controllers" file.
+	// excludes pseudo-controllers ("devices" and "freezer").
+	controllers map[string]struct{}
+}
+
+// NewManager creates a manager for cgroup v2 unified hierarchy.
+// dirPath is like "/sys/fs/cgroup/user.slice/user-1001.slice/session-1.scope".
+// If dirPath is empty, it is automatically set using config.
+func NewManager(config *configs.Cgroup, dirPath string) (cgroups.Manager, error) {
+	if dirPath == "" {
+		var err error
+		dirPath, err = defaultDirPath(config)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	m := &manager{
+		config:  config,
+		dirPath: dirPath,
+	}
+	return m, nil
+}
+
+func (m *manager) getControllers() error {
+	if m.controllers != nil {
+		return nil
+	}
+
+	data, err := cgroups.ReadFile(m.dirPath, "cgroup.controllers")
+	if err != nil {
+		if m.config.Rootless && m.config.Path == "" {
+			return nil
+		}
+		return err
+	}
+	fields := strings.Fields(data)
+	m.controllers = make(map[string]struct{}, len(fields))
+	for _, c := range fields {
+		m.controllers[c] = struct{}{}
+	}
+
+	return nil
+}
+
+func (m *manager) Apply(pid int) error {
+	if err := CreateCgroupPath(m.dirPath, m.config); err != nil {
+		// Related tests:
+		// - "runc create (no limits + no cgrouppath + no permission) succeeds"
+		// - "runc create (rootless + no limits + cgrouppath + no permission) fails with permission error"
+		// - "runc create (rootless + limits + no cgrouppath + no permission) fails with informative error"
+		if m.config.Rootless {
+			if m.config.Path == "" {
+				if blNeed, nErr := needAnyControllers(m.config.Resources); nErr == nil && !blNeed {
+					return nil
+				}
+				return fmt.Errorf("rootless needs no limits + no cgrouppath when no permission is granted for cgroups: %w", err)
+			}
+		}
+		return err
+	}
+	if err := cgroups.WriteCgroupProc(m.dirPath, pid); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *manager) GetPids() ([]int, error) {
+	return cgroups.GetPids(m.dirPath)
+}
+
+func (m *manager) GetAllPids() ([]int, error) {
+	return cgroups.GetAllPids(m.dirPath)
+}
+
+func (m *manager) GetStats() (*cgroups.Stats, error) {
+	var errs []error
+
+	st := cgroups.NewStats()
+
+	// pids (since kernel 4.5)
+	if err := statPids(m.dirPath, st); err != nil {
+		errs = append(errs, err)
+	}
+	// memory (since kernel 4.5)
+	if err := statMemory(m.dirPath, st); err != nil && !os.IsNotExist(err) {
+		errs = append(errs, err)
+	}
+	// io (since kernel 4.5)
+	if err := statIo(m.dirPath, st); err != nil && !os.IsNotExist(err) {
+		errs = append(errs, err)
+	}
+	// cpu (since kernel 4.15)
+	// Note cpu.stat is available even if the controller is not enabled.
+	if err := statCpu(m.dirPath, st); err != nil && !os.IsNotExist(err) {
+		errs = append(errs, err)
+	}
+	// hugetlb (since kernel 5.6)
+	if err := statHugeTlb(m.dirPath, st); err != nil && !os.IsNotExist(err) {
+		errs = append(errs, err)
+	}
+	// rdma (since kernel 4.11)
+	if err := fscommon.RdmaGetStats(m.dirPath, st); err != nil && !os.IsNotExist(err) {
+		errs = append(errs, err)
+	}
+	if len(errs) > 0 && !m.config.Rootless {
+		return st, fmt.Errorf("error while statting cgroup v2: %+v", errs)
+	}
+	return st, nil
+}
+
+func (m *manager) Freeze(state configs.FreezerState) error {
+	if m.config.Resources == nil {
+		return errors.New("cannot toggle freezer: cgroups not configured for container")
+	}
+	if err := setFreezer(m.dirPath, state); err != nil {
+		return err
+	}
+	m.config.Resources.Freezer = state
+	return nil
+}
+
+func (m *manager) Destroy() error {
+	return cgroups.RemovePath(m.dirPath)
+}
+
+func (m *manager) Path(_ string) string {
+	return m.dirPath
+}
+
+func (m *manager) Set(r *configs.Resources) error {
+	if r == nil {
+		return nil
+	}
+	if err := m.getControllers(); err != nil {
+		return err
+	}
+	// pids (since kernel 4.5)
+	if err := setPids(m.dirPath, r); err != nil {
+		return err
+	}
+	// memory (since kernel 4.5)
+	if err := setMemory(m.dirPath, r); err != nil {
+		return err
+	}
+	// io (since kernel 4.5)
+	if err := setIo(m.dirPath, r); err != nil {
+		return err
+	}
+	// cpu (since kernel 4.15)
+	if err := setCpu(m.dirPath, r); err != nil {
+		return err
+	}
+	// devices (since kernel 4.15, pseudo-controller)
+	//
+	// When rootless is true, errors from the device subsystem are ignored because it is really not expected to work.
+	// However, errors from other subsystems are not ignored.
+	// see @test "runc create (rootless + limits + no cgrouppath + no permission) fails with informative error"
+	if err := setDevices(m.dirPath, r); err != nil && !m.config.Rootless {
+		return err
+	}
+	// cpuset (since kernel 5.0)
+	if err := setCpuset(m.dirPath, r); err != nil {
+		return err
+	}
+	// hugetlb (since kernel 5.6)
+	if err := setHugeTlb(m.dirPath, r); err != nil {
+		return err
+	}
+	// rdma (since kernel 4.11)
+	if err := fscommon.RdmaSet(m.dirPath, r); err != nil {
+		return err
+	}
+	// freezer (since kernel 5.2, pseudo-controller)
+	if err := setFreezer(m.dirPath, r.Freezer); err != nil {
+		return err
+	}
+	if err := m.setUnified(r.Unified); err != nil {
+		return err
+	}
+	m.config.Resources = r
+	return nil
+}
+
+func (m *manager) setUnified(res map[string]string) error {
+	for k, v := range res {
+		if strings.Contains(k, "/") {
+			return fmt.Errorf("unified resource %q must be a file name (no slashes)", k)
+		}
+		if err := cgroups.WriteFile(m.dirPath, k, v); err != nil {
+			// Check for both EPERM and ENOENT since O_CREAT is used by WriteFile.
+			if errors.Is(err, os.ErrPermission) || errors.Is(err, os.ErrNotExist) {
+				// Check if a controller is available,
+				// to give more specific error if not.
+				sk := strings.SplitN(k, ".", 2)
+				if len(sk) != 2 {
+					return fmt.Errorf("unified resource %q must be in the form CONTROLLER.PARAMETER", k)
+				}
+				c := sk[0]
+				if _, ok := m.controllers[c]; !ok && c != "cgroup" {
+					return fmt.Errorf("unified resource %q can't be set: controller %q not available", k, c)
+				}
+			}
+			return fmt.Errorf("unable to set unified resource %q: %w", k, err)
+		}
+	}
+
+	return nil
+}
+
+func (m *manager) GetPaths() map[string]string {
+	paths := make(map[string]string, 1)
+	paths[""] = m.dirPath
+	return paths
+}
+
+func (m *manager) GetCgroups() (*configs.Cgroup, error) {
+	return m.config, nil
+}
+
+func (m *manager) GetFreezerState() (configs.FreezerState, error) {
+	return getFreezer(m.dirPath)
+}
+
+func (m *manager) Exists() bool {
+	return cgroups.PathExists(m.dirPath)
+}
+
+func OOMKillCount(path string) (uint64, error) {
+	return fscommon.GetValueByKey(path, "memory.events", "oom_kill")
+}
+
+func (m *manager) OOMKillCount() (uint64, error) {
+	c, err := OOMKillCount(m.dirPath)
+	if err != nil && m.config.Rootless && os.IsNotExist(err) {
+		err = nil
+	}
+
+	return c, err
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/hugetlb.go

@@ -0,0 +1,48 @@
+package fs2
+
+import (
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func isHugeTlbSet(r *configs.Resources) bool {
+	return len(r.HugetlbLimit) > 0
+}
+
+func setHugeTlb(dirPath string, r *configs.Resources) error {
+	if !isHugeTlbSet(r) {
+		return nil
+	}
+	for _, hugetlb := range r.HugetlbLimit {
+		if err := cgroups.WriteFile(dirPath, "hugetlb."+hugetlb.Pagesize+".max", strconv.FormatUint(hugetlb.Limit, 10)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func statHugeTlb(dirPath string, stats *cgroups.Stats) error {
+	hugetlbStats := cgroups.HugetlbStats{}
+	for _, pagesize := range cgroups.HugePageSizes() {
+		value, err := fscommon.GetCgroupParamUint(dirPath, "hugetlb."+pagesize+".current")
+		if err != nil {
+			return err
+		}
+		hugetlbStats.Usage = value
+
+		fileName := "hugetlb." + pagesize + ".events"
+		value, err = fscommon.GetValueByKey(dirPath, fileName, "max")
+		if err != nil {
+			return err
+		}
+		hugetlbStats.Failcnt = value
+
+		stats.HugetlbStats[pagesize] = hugetlbStats
+	}
+
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/io.go

@@ -0,0 +1,193 @@
+package fs2
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func isIoSet(r *configs.Resources) bool {
+	return r.BlkioWeight != 0 ||
+		len(r.BlkioWeightDevice) > 0 ||
+		len(r.BlkioThrottleReadBpsDevice) > 0 ||
+		len(r.BlkioThrottleWriteBpsDevice) > 0 ||
+		len(r.BlkioThrottleReadIOPSDevice) > 0 ||
+		len(r.BlkioThrottleWriteIOPSDevice) > 0
+}
+
+// bfqDeviceWeightSupported checks for per-device BFQ weight support (added
+// in kernel v5.4, commit 795fe54c2a8) by reading from "io.bfq.weight".
+func bfqDeviceWeightSupported(bfq *os.File) bool {
+	if bfq == nil {
+		return false
+	}
+	_, _ = bfq.Seek(0, 0)
+	buf := make([]byte, 32)
+	_, _ = bfq.Read(buf)
+	// If only a single number (default weight) if read back, we have older kernel.
+	_, err := strconv.ParseInt(string(bytes.TrimSpace(buf)), 10, 64)
+	return err != nil
+}
+
+func setIo(dirPath string, r *configs.Resources) error {
+	if !isIoSet(r) {
+		return nil
+	}
+
+	// If BFQ IO scheduler is available, use it.
+	var bfq *os.File
+	if r.BlkioWeight != 0 || len(r.BlkioWeightDevice) > 0 {
+		var err error
+		bfq, err = cgroups.OpenFile(dirPath, "io.bfq.weight", os.O_RDWR)
+		if err == nil {
+			defer bfq.Close()
+		} else if !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	if r.BlkioWeight != 0 {
+		if bfq != nil { // Use BFQ.
+			if _, err := bfq.WriteString(strconv.FormatUint(uint64(r.BlkioWeight), 10)); err != nil {
+				return err
+			}
+		} else {
+			// Fallback to io.weight with a conversion scheme.
+			v := cgroups.ConvertBlkIOToIOWeightValue(r.BlkioWeight)
+			if err := cgroups.WriteFile(dirPath, "io.weight", strconv.FormatUint(v, 10)); err != nil {
+				return err
+			}
+		}
+	}
+	if bfqDeviceWeightSupported(bfq) {
+		for _, wd := range r.BlkioWeightDevice {
+			if _, err := bfq.WriteString(wd.WeightString() + "\n"); err != nil {
+				return fmt.Errorf("setting device weight %q: %w", wd.WeightString(), err)
+			}
+		}
+	}
+	for _, td := range r.BlkioThrottleReadBpsDevice {
+		if err := cgroups.WriteFile(dirPath, "io.max", td.StringName("rbps")); err != nil {
+			return err
+		}
+	}
+	for _, td := range r.BlkioThrottleWriteBpsDevice {
+		if err := cgroups.WriteFile(dirPath, "io.max", td.StringName("wbps")); err != nil {
+			return err
+		}
+	}
+	for _, td := range r.BlkioThrottleReadIOPSDevice {
+		if err := cgroups.WriteFile(dirPath, "io.max", td.StringName("riops")); err != nil {
+			return err
+		}
+	}
+	for _, td := range r.BlkioThrottleWriteIOPSDevice {
+		if err := cgroups.WriteFile(dirPath, "io.max", td.StringName("wiops")); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func readCgroup2MapFile(dirPath string, name string) (map[string][]string, error) {
+	ret := map[string][]string{}
+	f, err := cgroups.OpenFile(dirPath, name, os.O_RDONLY)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		parts := strings.Fields(line)
+		if len(parts) < 2 {
+			continue
+		}
+		ret[parts[0]] = parts[1:]
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, &parseError{Path: dirPath, File: name, Err: err}
+	}
+	return ret, nil
+}
+
+func statIo(dirPath string, stats *cgroups.Stats) error {
+	const file = "io.stat"
+	values, err := readCgroup2MapFile(dirPath, file)
+	if err != nil {
+		return err
+	}
+	// more details on the io.stat file format: https://www.kernel.org/doc/Documentation/cgroup-v2.txt
+	var parsedStats cgroups.BlkioStats
+	for k, v := range values {
+		d := strings.Split(k, ":")
+		if len(d) != 2 {
+			continue
+		}
+		major, err := strconv.ParseUint(d[0], 10, 64)
+		if err != nil {
+			return &parseError{Path: dirPath, File: file, Err: err}
+		}
+		minor, err := strconv.ParseUint(d[1], 10, 64)
+		if err != nil {
+			return &parseError{Path: dirPath, File: file, Err: err}
+		}
+
+		for _, item := range v {
+			d := strings.Split(item, "=")
+			if len(d) != 2 {
+				continue
+			}
+			op := d[0]
+
+			// Map to the cgroupv1 naming and layout (in separate tables).
+			var targetTable *[]cgroups.BlkioStatEntry
+			switch op {
+			// Equivalent to cgroupv1's blkio.io_service_bytes.
+			case "rbytes":
+				op = "Read"
+				targetTable = &parsedStats.IoServiceBytesRecursive
+			case "wbytes":
+				op = "Write"
+				targetTable = &parsedStats.IoServiceBytesRecursive
+			// Equivalent to cgroupv1's blkio.io_serviced.
+			case "rios":
+				op = "Read"
+				targetTable = &parsedStats.IoServicedRecursive
+			case "wios":
+				op = "Write"
+				targetTable = &parsedStats.IoServicedRecursive
+			default:
+				// Skip over entries we cannot map to cgroupv1 stats for now.
+				// In the future we should expand the stats struct to include
+				// them.
+				logrus.Debugf("cgroupv2 io stats: skipping over unmappable %s entry", item)
+				continue
+			}
+
+			value, err := strconv.ParseUint(d[1], 10, 64)
+			if err != nil {
+				return &parseError{Path: dirPath, File: file, Err: err}
+			}
+
+			entry := cgroups.BlkioStatEntry{
+				Op:    op,
+				Major: major,
+				Minor: minor,
+				Value: value,
+			}
+			*targetTable = append(*targetTable, entry)
+		}
+	}
+	stats.BlkioStats = parsedStats
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/memory.go

@@ -0,0 +1,216 @@
+package fs2
+
+import (
+	"bufio"
+	"errors"
+	"math"
+	"os"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+// numToStr converts an int64 value to a string for writing to a
+// cgroupv2 files with .min, .max, .low, or .high suffix.
+// The value of -1 is converted to "max" for cgroupv1 compatibility
+// (which used to write -1 to remove the limit).
+func numToStr(value int64) (ret string) {
+	switch {
+	case value == 0:
+		ret = ""
+	case value == -1:
+		ret = "max"
+	default:
+		ret = strconv.FormatInt(value, 10)
+	}
+
+	return ret
+}
+
+func isMemorySet(r *configs.Resources) bool {
+	return r.MemoryReservation != 0 || r.Memory != 0 || r.MemorySwap != 0
+}
+
+func setMemory(dirPath string, r *configs.Resources) error {
+	if !isMemorySet(r) {
+		return nil
+	}
+	swap, err := cgroups.ConvertMemorySwapToCgroupV2Value(r.MemorySwap, r.Memory)
+	if err != nil {
+		return err
+	}
+	swapStr := numToStr(swap)
+	if swapStr == "" && swap == 0 && r.MemorySwap > 0 {
+		// memory and memorySwap set to the same value -- disable swap
+		swapStr = "0"
+	}
+	// never write empty string to `memory.swap.max`, it means set to 0.
+	if swapStr != "" {
+		if err := cgroups.WriteFile(dirPath, "memory.swap.max", swapStr); err != nil {
+			return err
+		}
+	}
+
+	if val := numToStr(r.Memory); val != "" {
+		if err := cgroups.WriteFile(dirPath, "memory.max", val); err != nil {
+			return err
+		}
+	}
+
+	// cgroup.Resources.KernelMemory is ignored
+
+	if val := numToStr(r.MemoryReservation); val != "" {
+		if err := cgroups.WriteFile(dirPath, "memory.low", val); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func statMemory(dirPath string, stats *cgroups.Stats) error {
+	const file = "memory.stat"
+	statsFile, err := cgroups.OpenFile(dirPath, file, os.O_RDONLY)
+	if err != nil {
+		return err
+	}
+	defer statsFile.Close()
+
+	sc := bufio.NewScanner(statsFile)
+	for sc.Scan() {
+		t, v, err := fscommon.ParseKeyValue(sc.Text())
+		if err != nil {
+			return &parseError{Path: dirPath, File: file, Err: err}
+		}
+		stats.MemoryStats.Stats[t] = v
+	}
+	if err := sc.Err(); err != nil {
+		return &parseError{Path: dirPath, File: file, Err: err}
+	}
+	stats.MemoryStats.Cache = stats.MemoryStats.Stats["file"]
+	// Unlike cgroup v1 which has memory.use_hierarchy binary knob,
+	// cgroup v2 is always hierarchical.
+	stats.MemoryStats.UseHierarchy = true
+
+	memoryUsage, err := getMemoryDataV2(dirPath, "")
+	if err != nil {
+		if errors.Is(err, unix.ENOENT) && dirPath == UnifiedMountpoint {
+			// The root cgroup does not have memory.{current,max}
+			// so emulate those using data from /proc/meminfo.
+			return statsFromMeminfo(stats)
+		}
+		return err
+	}
+	stats.MemoryStats.Usage = memoryUsage
+	swapUsage, err := getMemoryDataV2(dirPath, "swap")
+	if err != nil {
+		return err
+	}
+	// As cgroup v1 reports SwapUsage values as mem+swap combined,
+	// while in cgroup v2 swap values do not include memory,
+	// report combined mem+swap for v1 compatibility.
+	swapUsage.Usage += memoryUsage.Usage
+	if swapUsage.Limit != math.MaxUint64 {
+		swapUsage.Limit += memoryUsage.Limit
+	}
+	stats.MemoryStats.SwapUsage = swapUsage
+
+	return nil
+}
+
+func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) {
+	memoryData := cgroups.MemoryData{}
+
+	moduleName := "memory"
+	if name != "" {
+		moduleName = "memory." + name
+	}
+	usage := moduleName + ".current"
+	limit := moduleName + ".max"
+
+	value, err := fscommon.GetCgroupParamUint(path, usage)
+	if err != nil {
+		if name != "" && os.IsNotExist(err) {
+			// Ignore EEXIST as there's no swap accounting
+			// if kernel CONFIG_MEMCG_SWAP is not set or
+			// swapaccount=0 kernel boot parameter is given.
+			return cgroups.MemoryData{}, nil
+		}
+		return cgroups.MemoryData{}, err
+	}
+	memoryData.Usage = value
+
+	value, err = fscommon.GetCgroupParamUint(path, limit)
+	if err != nil {
+		return cgroups.MemoryData{}, err
+	}
+	memoryData.Limit = value
+
+	return memoryData, nil
+}
+
+func statsFromMeminfo(stats *cgroups.Stats) error {
+	const file = "/proc/meminfo"
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Fields we are interested in.
+	var (
+		swap_free  uint64
+		swap_total uint64
+		main_total uint64
+		main_free  uint64
+	)
+	mem := map[string]*uint64{
+		"SwapFree":  &swap_free,
+		"SwapTotal": &swap_total,
+		"MemTotal":  &main_total,
+		"MemFree":   &main_free,
+	}
+
+	found := 0
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		parts := strings.SplitN(sc.Text(), ":", 3)
+		if len(parts) != 2 {
+			// Should not happen.
+			continue
+		}
+		k := parts[0]
+		p, ok := mem[k]
+		if !ok {
+			// Unknown field -- not interested.
+			continue
+		}
+		vStr := strings.TrimSpace(strings.TrimSuffix(parts[1], " kB"))
+		*p, err = strconv.ParseUint(vStr, 10, 64)
+		if err != nil {
+			return &parseError{File: file, Err: errors.New("bad value for " + k)}
+		}
+
+		found++
+		if found == len(mem) {
+			// Got everything we need -- skip the rest.
+			break
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return &parseError{Path: "", File: file, Err: err}
+	}
+
+	stats.MemoryStats.SwapUsage.Usage = (swap_total - swap_free) * 1024
+	stats.MemoryStats.SwapUsage.Limit = math.MaxUint64
+
+	stats.MemoryStats.Usage.Usage = (main_total - main_free) * 1024
+	stats.MemoryStats.Usage.Limit = math.MaxUint64
+
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs2/pids.go

@@ -0,0 +1,72 @@
+package fs2
+
+import (
+	"errors"
+	"math"
+	"os"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func isPidsSet(r *configs.Resources) bool {
+	return r.PidsLimit != 0
+}
+
+func setPids(dirPath string, r *configs.Resources) error {
+	if !isPidsSet(r) {
+		return nil
+	}
+	if val := numToStr(r.PidsLimit); val != "" {
+		if err := cgroups.WriteFile(dirPath, "pids.max", val); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func statPidsFromCgroupProcs(dirPath string, stats *cgroups.Stats) error {
+	// if the controller is not enabled, let's read PIDS from cgroups.procs
+	// (or threads if cgroup.threads is enabled)
+	contents, err := cgroups.ReadFile(dirPath, "cgroup.procs")
+	if errors.Is(err, unix.ENOTSUP) {
+		contents, err = cgroups.ReadFile(dirPath, "cgroup.threads")
+	}
+	if err != nil {
+		return err
+	}
+	pids := strings.Count(contents, "\n")
+	stats.PidsStats.Current = uint64(pids)
+	stats.PidsStats.Limit = 0
+	return nil
+}
+
+func statPids(dirPath string, stats *cgroups.Stats) error {
+	current, err := fscommon.GetCgroupParamUint(dirPath, "pids.current")
+	if err != nil {
+		if os.IsNotExist(err) {
+			return statPidsFromCgroupProcs(dirPath, stats)
+		}
+		return err
+	}
+
+	max, err := fscommon.GetCgroupParamUint(dirPath, "pids.max")
+	if err != nil {
+		return err
+	}
+	// If no limit is set, read from pids.max returns "max", which is
+	// converted to MaxUint64 by GetCgroupParamUint. Historically, we
+	// represent "no limit" for pids as 0, thus this conversion.
+	if max == math.MaxUint64 {
+		max = 0
+	}
+
+	stats.PidsStats.Current = current
+	stats.PidsStats.Limit = max
+	return nil
+}