Merge pull request #4296 from dims/revendor-containerd-cri-for-tolerating-hugepages-cgroup

Revendor CRI to 62c91260d2

vendor.conf

@@ -56,7 +56,7 @@ gotest.tools/v3                                     v3.0.2
 github.com/cilium/ebpf                              4032b1d8aae306b7bb94a2a11002932caf88c644
 
 # cri dependencies
-github.com/containerd/cri                           8898550e348932e406049e937d98fb7564ac4e7a # master
+github.com/containerd/cri                           62c91260d2f43b57fff408a9263a800b7a06a647 # master
 github.com/davecgh/go-spew                          v1.1.1
 github.com/docker/docker                            4634ce647cf2ce2c6031129ccd109e557244986f
 github.com/docker/spdystream                        449fdfce4d962303d702fec724ef0ad181c92528

vendor/github.com/containerd/cri/pkg/config/config.go

@@ -230,6 +230,10 @@ type PluginConfig struct {
 	// UnsetSeccompProfile is the profile containerd/cri will use If the provided seccomp profile is
 	// unset (`""`) for a container (default is `unconfined`)
 	UnsetSeccompProfile string `toml:"unset_seccomp_profile" json:"unsetSeccompProfile"`
+	// TolerateMissingHugePagesCgroupController if set to false will error out on create/update
+	// container requests with huge page limits if the cgroup controller for hugepages is not present.
+	// This helps with supporting Kubernetes <=1.18 out of the box. (default is `true`)
+	TolerateMissingHugePagesCgroupController bool `toml:"tolerate_missing_hugepages_controller" json:"tolerateMissingHugePagesCgroupController"`
 }
 
 // X509KeyPairStreaming contains the x509 configuration for streaming

vendor/github.com/containerd/cri/pkg/config/config_unix.go

@@ -63,7 +63,8 @@ func DefaultConfig() PluginConfig {
 				},
 			},
 		},
-		MaxConcurrentDownloads: 3,
+		MaxConcurrentDownloads:                   3,
-		DisableProcMount:       false,
+		DisableProcMount:                         false,
+		TolerateMissingHugePagesCgroupController: true,
 	}
 }

vendor/github.com/containerd/cri/pkg/containerd/opts/spec_unix.go

@@ -27,6 +27,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
+	"syscall"
 
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/log"
@@ -36,6 +38,7 @@ import (
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 
@@ -405,7 +408,7 @@ func WithSelinuxLabels(process, mount string) oci.SpecOpts {
 }
 
 // WithResources sets the provided resource restrictions
-func WithResources(resources *runtime.LinuxContainerResources) oci.SpecOpts {
+func WithResources(resources *runtime.LinuxContainerResources, tolerateMissingHugePagesCgroupController bool) oci.SpecOpts {
 	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
 		if resources == nil {
 			return nil
@@ -448,16 +451,93 @@ func WithResources(resources *runtime.LinuxContainerResources) oci.SpecOpts {
 		if limit != 0 {
 			s.Linux.Resources.Memory.Limit = &limit
 		}
-		for _, limit := range hugepages {
+		if isHugePagesControllerPresent() {
-			s.Linux.Resources.HugepageLimits = append(s.Linux.Resources.HugepageLimits, runtimespec.LinuxHugepageLimit{
+			for _, limit := range hugepages {
-				Pagesize: limit.PageSize,
+				s.Linux.Resources.HugepageLimits = append(s.Linux.Resources.HugepageLimits, runtimespec.LinuxHugepageLimit{
-				Limit:    limit.Limit,
+					Pagesize: limit.PageSize,
-			})
+					Limit:    limit.Limit,
+				})
+			}
+		} else {
+			if !tolerateMissingHugePagesCgroupController {
+				return errors.Errorf("huge pages limits are specified but hugetlb cgroup controller is missing. " +
+					"Please set tolerate_missing_hugepages_controller to `true` to ignore this error")
+			}
+			logrus.Warn("hugetlb cgroup controller is absent. skipping huge pages limits")
 		}
 		return nil
 	}
 }
 
+var (
+	supportsHugetlbOnce sync.Once
+	supportsHugetlb     bool
+)
+
+func isHugePagesControllerPresent() bool {
+	supportsHugetlbOnce.Do(func() {
+		supportsHugetlb = false
+		if IsCgroup2UnifiedMode() {
+			supportsHugetlb, _ = cgroupv2HasHugetlb()
+		} else {
+			supportsHugetlb, _ = cgroupv1HasHugetlb()
+		}
+	})
+	return supportsHugetlb
+}
+
+var (
+	_cgroupv1HasHugetlbOnce sync.Once
+	_cgroupv1HasHugetlb     bool
+	_cgroupv1HasHugetlbErr  error
+	_cgroupv2HasHugetlbOnce sync.Once
+	_cgroupv2HasHugetlb     bool
+	_cgroupv2HasHugetlbErr  error
+	isUnifiedOnce           sync.Once
+	isUnified               bool
+)
+
+// cgroupv1HasHugetlb returns whether the hugetlb controller is present on
+// cgroup v1.
+func cgroupv1HasHugetlb() (bool, error) {
+	_cgroupv1HasHugetlbOnce.Do(func() {
+		if _, err := ioutil.ReadDir("/sys/fs/cgroup/hugetlb"); err != nil {
+			_cgroupv1HasHugetlbErr = errors.Wrap(err, "readdir /sys/fs/cgroup/hugetlb")
+			_cgroupv1HasHugetlb = false
+		} else {
+			_cgroupv1HasHugetlbErr = nil
+			_cgroupv1HasHugetlb = true
+		}
+	})
+	return _cgroupv1HasHugetlb, _cgroupv1HasHugetlbErr
+}
+
+// cgroupv2HasHugetlb returns whether the hugetlb controller is present on
+// cgroup v2.
+func cgroupv2HasHugetlb() (bool, error) {
+	_cgroupv2HasHugetlbOnce.Do(func() {
+		controllers, err := ioutil.ReadFile("/sys/fs/cgroup/cgroup.controllers")
+		if err != nil {
+			_cgroupv2HasHugetlbErr = errors.Wrap(err, "read /sys/fs/cgroup/cgroup.controllers")
+			return
+		}
+		_cgroupv2HasHugetlb = strings.Contains(string(controllers), "hugetlb")
+	})
+	return _cgroupv2HasHugetlb, _cgroupv2HasHugetlbErr
+}
+
+// IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode.
+func IsCgroup2UnifiedMode() bool {
+	isUnifiedOnce.Do(func() {
+		var st syscall.Statfs_t
+		if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil {
+			panic("cannot statfs cgroup root")
+		}
+		isUnified = st.Type == unix.CGROUP2_SUPER_MAGIC
+	})
+	return isUnified
+}
+
 // WithOOMScoreAdj sets the oom score
 func WithOOMScoreAdj(config *runtime.ContainerConfig, restrict bool) oci.SpecOpts {
 	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {

vendor/github.com/containerd/cri/pkg/server/container_create_unix.go

@@ -225,7 +225,7 @@ func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint3
 	if c.config.DisableCgroup {
 		specOpts = append(specOpts, customopts.WithDisabledCgroups)
 	} else {
-		specOpts = append(specOpts, customopts.WithResources(config.GetLinux().GetResources()))
+		specOpts = append(specOpts, customopts.WithResources(config.GetLinux().GetResources(), c.config.TolerateMissingHugePagesCgroupController))
 		if sandboxConfig.GetLinux().GetCgroupParent() != "" {
 			cgroupsPath := getCgroupsPath(sandboxConfig.GetLinux().GetCgroupParent(), id)
 			specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))

vendor/github.com/containerd/cri/pkg/server/container_update_resources_unix.go

@@ -72,7 +72,8 @@ func (c *criService) updateContainerResources(ctx context.Context,
 	if err != nil {
 		return errors.Wrap(err, "failed to get container spec")
 	}
-	newSpec, err := updateOCILinuxResource(ctx, oldSpec, resources)
+	newSpec, err := updateOCILinuxResource(ctx, oldSpec, resources,
+		c.config.TolerateMissingHugePagesCgroupController)
 	if err != nil {
 		return errors.Wrap(err, "failed to update resource in spec")
 	}
@@ -132,7 +133,8 @@ func updateContainerSpec(ctx context.Context, cntr containerd.Container, spec *r
 }
 
 // updateOCILinuxResource updates container resource limit.
-func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *runtime.LinuxContainerResources) (*runtimespec.Spec, error) {
+func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *runtime.LinuxContainerResources,
+	tolerateMissingHugePagesCgroupController bool) (*runtimespec.Spec, error) {
 	// Copy to make sure old spec is not changed.
 	var cloned runtimespec.Spec
 	if err := util.DeepCopy(&cloned, spec); err != nil {
@@ -141,7 +143,7 @@ func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *ru
 	if cloned.Linux == nil {
 		cloned.Linux = &runtimespec.Linux{}
 	}
-	if err := opts.WithResources(new)(ctx, nil, nil, &cloned); err != nil {
+	if err := opts.WithResources(new, tolerateMissingHugePagesCgroupController)(ctx, nil, nil, &cloned); err != nil {
 		return nil, errors.Wrap(err, "unable to set linux container resources")
 	}
 	return &cloned, nil