Merge pull request #123593 from giuseppe/userns-use-kubelet-user-mappings
KEP-127: kubelet: honor kubelet user mappings
This commit is contained in:
		| @@ -823,6 +823,7 @@ const ( | ||||
| 	// owner: @rata, @giuseppe | ||||
| 	// kep: https://kep.k8s.io/127 | ||||
| 	// alpha: v1.25 | ||||
| 	// beta: v1.30 | ||||
| 	// | ||||
| 	// Enables user namespace support for stateless pods. | ||||
| 	UserNamespacesSupport featuregate.Feature = "UserNamespacesSupport" | ||||
| @@ -1154,7 +1155,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS | ||||
|  | ||||
| 	VolumeCapacityPriority: {Default: false, PreRelease: featuregate.Alpha}, | ||||
|  | ||||
| 	UserNamespacesSupport: {Default: false, PreRelease: featuregate.Alpha}, | ||||
| 	UserNamespacesSupport: {Default: false, PreRelease: featuregate.Beta}, | ||||
|  | ||||
| 	WinDSR: {Default: false, PreRelease: featuregate.Alpha}, | ||||
|  | ||||
|   | ||||
| @@ -123,6 +123,15 @@ func (kl *Kubelet) HandlerSupportsUserNamespaces(rtHandler string) (bool, error) | ||||
| 	return h.SupportsUserNamespaces, nil | ||||
| } | ||||
|  | ||||
| // GetKubeletMappings gets the additional IDs allocated for the Kubelet. | ||||
| func (kl *Kubelet) GetKubeletMappings() (uint32, uint32, error) { | ||||
| 	return kl.getKubeletMappings() | ||||
| } | ||||
|  | ||||
| func (kl *Kubelet) GetMaxPods() int { | ||||
| 	return kl.maxPods | ||||
| } | ||||
|  | ||||
| // getPodDir returns the full path to the per-pod directory for the pod with | ||||
| // the given UID. | ||||
| func (kl *Kubelet) getPodDir(podUID types.UID) string { | ||||
|   | ||||
| @@ -19,14 +19,18 @@ package kubelet | ||||
| import ( | ||||
| 	"bytes" | ||||
| 	"context" | ||||
| 	goerrors "errors" | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"net/http" | ||||
| 	"net/url" | ||||
| 	"os" | ||||
| 	"os/exec" | ||||
| 	"os/user" | ||||
| 	"path/filepath" | ||||
| 	"runtime" | ||||
| 	"sort" | ||||
| 	"strconv" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/google/go-cmp/cmp" | ||||
| @@ -76,8 +80,90 @@ const ( | ||||
| const ( | ||||
| 	PodInitializing   = "PodInitializing" | ||||
| 	ContainerCreating = "ContainerCreating" | ||||
|  | ||||
| 	kubeletUser = "kubelet" | ||||
| ) | ||||
|  | ||||
| // parseGetSubIdsOutput parses the output from the `getsubids` tool, which is used to query subordinate user or group ID ranges for | ||||
| // a given user or group. getsubids produces a line for each mapping configured. | ||||
| // Here we expect that there is a single mapping, and the same values are used for the subordinate user and group ID ranges. | ||||
| // The output is something like: | ||||
| // $ getsubids kubelet | ||||
| // 0: kubelet 65536 2147483648 | ||||
| // $ getsubids -g kubelet | ||||
| // 0: kubelet 65536 2147483648 | ||||
| func parseGetSubIdsOutput(input string) (uint32, uint32, error) { | ||||
| 	lines := strings.Split(strings.Trim(input, "\n"), "\n") | ||||
| 	if len(lines) != 1 { | ||||
| 		return 0, 0, fmt.Errorf("error parsing line %q: it must contain only one line", input) | ||||
| 	} | ||||
|  | ||||
| 	parts := strings.Fields(lines[0]) | ||||
| 	if len(parts) != 4 { | ||||
| 		return 0, 0, fmt.Errorf("invalid line %q", input) | ||||
| 	} | ||||
|  | ||||
| 	// Parsing the numbers | ||||
| 	num1, err := strconv.ParseUint(parts[2], 10, 32) | ||||
| 	if err != nil { | ||||
| 		return 0, 0, fmt.Errorf("error parsing line %q: %w", input, err) | ||||
| 	} | ||||
|  | ||||
| 	num2, err := strconv.ParseUint(parts[3], 10, 32) | ||||
| 	if err != nil { | ||||
| 		return 0, 0, fmt.Errorf("error parsing line %q: %w", input, err) | ||||
| 	} | ||||
|  | ||||
| 	return uint32(num1), uint32(num2), nil | ||||
| } | ||||
|  | ||||
| // getKubeletMappings returns the range of IDs that can be used to configure user namespaces. | ||||
| // If subordinate user or group ID ranges are specified for the kubelet user and the getsubids tool | ||||
| // is installed, then the single mapping specified both for user and group IDs will be used. | ||||
| // If the tool is not installed, or there are no IDs configured, the default mapping is returned. | ||||
| // The default mapping includes the entire IDs range except IDs below 65536. | ||||
| func (kl *Kubelet) getKubeletMappings() (uint32, uint32, error) { | ||||
| 	// default mappings to return if there is no specific configuration | ||||
| 	const defaultFirstID = 1 << 16 | ||||
| 	const defaultLen = 1<<32 - defaultFirstID | ||||
|  | ||||
| 	if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesSupport) { | ||||
| 		return defaultFirstID, defaultLen, nil | ||||
| 	} | ||||
|  | ||||
| 	_, err := user.Lookup(kubeletUser) | ||||
| 	if err != nil { | ||||
| 		var unknownUserErr user.UnknownUserError | ||||
| 		if goerrors.As(err, &unknownUserErr) { | ||||
| 			// if the user is not found, we assume that the user is not configured | ||||
| 			return defaultFirstID, defaultLen, nil | ||||
| 		} | ||||
| 		return 0, 0, err | ||||
| 	} | ||||
|  | ||||
| 	execName := "getsubids" | ||||
| 	cmd, err := exec.LookPath(execName) | ||||
| 	if err != nil { | ||||
| 		if os.IsNotExist(err) { | ||||
| 			klog.V(2).InfoS("Could not find executable, default mappings will be used for the user namespaces", "executable", execName, "err", err) | ||||
| 			return defaultFirstID, defaultLen, nil | ||||
| 		} | ||||
| 		return 0, 0, err | ||||
| 	} | ||||
| 	outUids, err := exec.Command(cmd, kubeletUser).Output() | ||||
| 	if err != nil { | ||||
| 		return 0, 0, fmt.Errorf("error retrieving additional ids for user %q", kubeletUser) | ||||
| 	} | ||||
| 	outGids, err := exec.Command(cmd, "-g", kubeletUser).Output() | ||||
| 	if err != nil { | ||||
| 		return 0, 0, fmt.Errorf("error retrieving additional gids for user %q", kubeletUser) | ||||
| 	} | ||||
| 	if string(outUids) != string(outGids) { | ||||
| 		return 0, 0, fmt.Errorf("mismatched subuids and subgids for user %q", kubeletUser) | ||||
| 	} | ||||
| 	return parseGetSubIdsOutput(string(outUids)) | ||||
| } | ||||
|  | ||||
| // Get a list of pods that have data directories. | ||||
| func (kl *Kubelet) listPodsFromDisk() ([]types.UID, error) { | ||||
| 	podInfos, err := os.ReadDir(kl.getPodsDir()) | ||||
|   | ||||
| @@ -6013,3 +6013,77 @@ func TestGetNonExistentImagePullSecret(t *testing.T) { | ||||
| 	event := <-fakeRecorder.Events | ||||
| 	assert.Equal(t, event, expectedEvent) | ||||
| } | ||||
|  | ||||
| func TestParseGetSubIdsOutput(t *testing.T) { | ||||
| 	tests := []struct { | ||||
| 		name         string | ||||
| 		input        string | ||||
| 		wantFirstID  uint32 | ||||
| 		wantRangeLen uint32 | ||||
| 		wantErr      bool | ||||
| 	}{ | ||||
| 		{ | ||||
| 			name:         "valid", | ||||
| 			input:        "0: kubelet 65536 2147483648", | ||||
| 			wantFirstID:  65536, | ||||
| 			wantRangeLen: 2147483648, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "multiple lines", | ||||
| 			input:   "0: kubelet 1 2\n1: kubelet 3 4\n", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "wrong format", | ||||
| 			input:   "0: kubelet 65536", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "non numeric 1", | ||||
| 			input:   "0: kubelet Foo 65536", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "non numeric 2", | ||||
| 			input:   "0: kubelet 0 Bar", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "overflow 1", | ||||
| 			input:   "0: kubelet 4294967296 2147483648", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "overflow 2", | ||||
| 			input:   "0: kubelet 65536 4294967296", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "negative value 1", | ||||
| 			input:   "0: kubelet -1 2147483648", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:    "negative value 2", | ||||
| 			input:   "0: kubelet 65536 -1", | ||||
| 			wantErr: true, | ||||
| 		}, | ||||
| 	} | ||||
| 	for _, tc := range tests { | ||||
| 		t.Run(tc.name, func(t *testing.T) { | ||||
| 			gotFirstID, gotRangeLen, err := parseGetSubIdsOutput(tc.input) | ||||
| 			if tc.wantErr { | ||||
| 				if err == nil { | ||||
| 					t.Errorf("%s: expected error, got nil", tc.name) | ||||
| 				} | ||||
| 			} else { | ||||
| 				if err != nil { | ||||
| 					t.Errorf("%s: unexpected error: %v", tc.name, err) | ||||
| 				} | ||||
| 				if gotFirstID != tc.wantFirstID || gotRangeLen != tc.wantRangeLen { | ||||
| 					t.Errorf("%s: got (%d, %d), want (%d, %d)", tc.name, gotFirstID, gotRangeLen, tc.wantFirstID, tc.wantRangeLen) | ||||
| 				} | ||||
| 			} | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|   | ||||
| @@ -19,7 +19,6 @@ package userns | ||||
| import ( | ||||
| 	"encoding/json" | ||||
| 	"fmt" | ||||
| 	"math" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"sync" | ||||
| @@ -40,10 +39,6 @@ import ( | ||||
| // length for the user namespace to create (65536). | ||||
| const userNsLength = (1 << 16) | ||||
|  | ||||
| // Limit the total number of pods using userns in this node to this value. | ||||
| // This is an alpha limitation that will probably be lifted later. | ||||
| const maxPods = 1024 | ||||
|  | ||||
| // Create a new map when we removed enough pods to avoid memory leaks | ||||
| // since Go maps never free memory. | ||||
| const mapReInitializeThreshold = 1000 | ||||
| @@ -52,14 +47,19 @@ type userNsPodsManager interface { | ||||
| 	HandlerSupportsUserNamespaces(runtimeHandler string) (bool, error) | ||||
| 	GetPodDir(podUID types.UID) string | ||||
| 	ListPodsFromDisk() ([]types.UID, error) | ||||
| 	GetKubeletMappings() (uint32, uint32, error) | ||||
| 	GetMaxPods() int | ||||
| } | ||||
|  | ||||
| type UsernsManager struct { | ||||
| 	used         *allocator.AllocationBitmap | ||||
| 	usedBy       map[types.UID]uint32 // Map pod.UID to range used | ||||
| 	removed      int | ||||
| 	numAllocated int | ||||
| 	kl           userNsPodsManager | ||||
| 	used    *allocator.AllocationBitmap | ||||
| 	usedBy  map[types.UID]uint32 // Map pod.UID to range used | ||||
| 	removed int | ||||
|  | ||||
| 	off int | ||||
| 	len int | ||||
|  | ||||
| 	kl userNsPodsManager | ||||
| 	// This protects all members except for kl.anager | ||||
| 	lock sync.Mutex | ||||
| } | ||||
| @@ -130,16 +130,33 @@ func (m *UsernsManager) readMappingsFromFile(pod types.UID) ([]byte, error) { | ||||
| } | ||||
|  | ||||
| func MakeUserNsManager(kl userNsPodsManager) (*UsernsManager, error) { | ||||
| 	kubeletMappingID, kubeletMappingLen, err := kl.GetKubeletMappings() | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	if kubeletMappingID%userNsLength != 0 { | ||||
| 		return nil, fmt.Errorf("kubelet user assigned ID %v is not a multiple of %v", kubeletMappingID, userNsLength) | ||||
| 	} | ||||
| 	if kubeletMappingID < userNsLength { | ||||
| 		// We don't allow to map 0, as security is circumvented. | ||||
| 		return nil, fmt.Errorf("kubelet user assigned ID %v must be greater or equal to %v", kubeletMappingID, userNsLength) | ||||
| 	} | ||||
| 	if kubeletMappingLen%userNsLength != 0 { | ||||
| 		return nil, fmt.Errorf("kubelet user assigned IDs length %v is not a multiple of %v", kubeletMappingLen, userNsLength) | ||||
| 	} | ||||
| 	if kubeletMappingLen/userNsLength < uint32(kl.GetMaxPods()) { | ||||
| 		return nil, fmt.Errorf("kubelet user assigned IDs are not enough to support %v pods", kl.GetMaxPods()) | ||||
| 	} | ||||
| 	off := int(kubeletMappingID / userNsLength) | ||||
| 	len := int(kubeletMappingLen / userNsLength) | ||||
|  | ||||
| 	m := UsernsManager{ | ||||
| 		// Create a bitArray for all the UID space (2^32). | ||||
| 		// As a by product of that, no index param to bitArray can be out of bounds (index is uint32). | ||||
| 		used:   allocator.NewAllocationMap((math.MaxUint32+1)/userNsLength, "user namespaces"), | ||||
| 		used:   allocator.NewAllocationMap(len, "user namespaces"), | ||||
| 		usedBy: make(map[types.UID]uint32), | ||||
| 		kl:     kl, | ||||
| 	} | ||||
| 	// First block is reserved for the host. | ||||
| 	if _, err := m.used.Allocate(0); err != nil { | ||||
| 		return nil, err | ||||
| 		off:    off, | ||||
| 		len:    len, | ||||
| 	} | ||||
|  | ||||
| 	// do not bother reading the list of pods if user namespaces are not enabled. | ||||
| @@ -184,7 +201,10 @@ func (m *UsernsManager) recordPodMappings(pod types.UID) error { | ||||
|  | ||||
| // isSet checks if the specified index is already set. | ||||
| func (m *UsernsManager) isSet(v uint32) bool { | ||||
| 	index := int(v / userNsLength) | ||||
| 	index := int(v/userNsLength) - m.off | ||||
| 	if index < 0 || index >= m.len { | ||||
| 		return true | ||||
| 	} | ||||
| 	return m.used.Has(index) | ||||
| } | ||||
|  | ||||
| @@ -192,16 +212,6 @@ func (m *UsernsManager) isSet(v uint32) bool { | ||||
| // The first return value is the first ID in the user namespace, the second returns | ||||
| // the length for the user namespace range. | ||||
| func (m *UsernsManager) allocateOne(pod types.UID) (firstID uint32, length uint32, err error) { | ||||
| 	if m.numAllocated >= maxPods { | ||||
| 		return 0, 0, fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated) | ||||
| 	} | ||||
| 	m.numAllocated++ | ||||
| 	defer func() { | ||||
| 		if err != nil { | ||||
| 			m.numAllocated-- | ||||
| 		} | ||||
| 	}() | ||||
|  | ||||
| 	firstZero, found, err := m.used.AllocateNext() | ||||
| 	if err != nil { | ||||
| 		return 0, 0, err | ||||
| @@ -212,7 +222,7 @@ func (m *UsernsManager) allocateOne(pod types.UID) (firstID uint32, length uint3 | ||||
|  | ||||
| 	klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod) | ||||
|  | ||||
| 	firstID = uint32(firstZero * userNsLength) | ||||
| 	firstID = uint32((firstZero + m.off) * userNsLength) | ||||
| 	m.usedBy[pod] = firstID | ||||
| 	return firstID, userNsLength, nil | ||||
| } | ||||
| @@ -229,7 +239,10 @@ func (m *UsernsManager) record(pod types.UID, from, length uint32) (err error) { | ||||
| 	if found && prevFrom != from { | ||||
| 		return fmt.Errorf("different user namespace range already used by pod %q", pod) | ||||
| 	} | ||||
| 	index := int(from / userNsLength) | ||||
| 	index := int(from/userNsLength) - m.off | ||||
| 	if index < 0 || index >= m.len { | ||||
| 		return fmt.Errorf("id %v is out of range", from) | ||||
| 	} | ||||
| 	// if the pod wasn't found then verify the range is free. | ||||
| 	if !found && m.used.Has(index) { | ||||
| 		return fmt.Errorf("range picked for pod %q already taken", pod) | ||||
| @@ -238,15 +251,6 @@ func (m *UsernsManager) record(pod types.UID, from, length uint32) (err error) { | ||||
| 	if found && prevFrom == from { | ||||
| 		return nil | ||||
| 	} | ||||
| 	if m.numAllocated >= maxPods { | ||||
| 		return fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated) | ||||
| 	} | ||||
| 	m.numAllocated++ | ||||
| 	defer func() { | ||||
| 		if err != nil { | ||||
| 			m.numAllocated-- | ||||
| 		} | ||||
| 	}() | ||||
|  | ||||
| 	klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod) | ||||
|  | ||||
| @@ -291,7 +295,6 @@ func (m *UsernsManager) releaseWithLock(pod types.UID) { | ||||
| 	delete(m.usedBy, pod) | ||||
|  | ||||
| 	klog.V(5).InfoS("releasing pod user namespace allocation", "podUID", pod) | ||||
| 	m.numAllocated-- | ||||
| 	m.removed++ | ||||
|  | ||||
| 	_ = os.Remove(filepath.Join(m.kl.GetPodDir(pod), mappingsFile)) | ||||
| @@ -304,7 +307,7 @@ func (m *UsernsManager) releaseWithLock(pod types.UID) { | ||||
| 		m.usedBy = n | ||||
| 		m.removed = 0 | ||||
| 	} | ||||
| 	m.used.Release(int(v / userNsLength)) | ||||
| 	_ = m.used.Release(int(v/userNsLength) - m.off) | ||||
| } | ||||
|  | ||||
| func (m *UsernsManager) parseUserNsFileAndRecord(pod types.UID, content []byte) (userNs userNamespace, err error) { | ||||
|   | ||||
							
								
								
									
										137
									
								
								pkg/kubelet/userns/userns_manager_switch_test.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										137
									
								
								pkg/kubelet/userns/userns_manager_switch_test.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,137 @@ | ||||
| /* | ||||
| Copyright 2024 The Kubernetes Authors. | ||||
|  | ||||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| you may not use this file except in compliance with the License. | ||||
| You may obtain a copy of the License at | ||||
|  | ||||
|     http://www.apache.org/licenses/LICENSE-2.0 | ||||
|  | ||||
| Unless required by applicable law or agreed to in writing, software | ||||
| distributed under the License is distributed on an "AS IS" BASIS, | ||||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||||
| See the License for the specific language governing permissions and | ||||
| limitations under the License. | ||||
| */ | ||||
|  | ||||
| package userns | ||||
|  | ||||
| import ( | ||||
| 	"testing" | ||||
|  | ||||
| 	"github.com/stretchr/testify/assert" | ||||
| 	"github.com/stretchr/testify/require" | ||||
| 	v1 "k8s.io/api/core/v1" | ||||
| 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||||
| 	"k8s.io/apimachinery/pkg/types" | ||||
| 	utilfeature "k8s.io/apiserver/pkg/util/feature" | ||||
| 	featuregatetesting "k8s.io/component-base/featuregate/testing" | ||||
| 	pkgfeatures "k8s.io/kubernetes/pkg/features" | ||||
| ) | ||||
|  | ||||
| func TestMakeUserNsManagerSwitch(t *testing.T) { | ||||
| 	// Create the manager with the feature gate enabled, to record some pods on disk. | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| 	pods := []types.UID{"pod-1", "pod-2"} | ||||
|  | ||||
| 	testUserNsPodsManager := &testUserNsPodsManager{ | ||||
| 		podDir: t.TempDir(), | ||||
| 		// List the same pods we will record, so the second time we create the userns | ||||
| 		// manager, it will find these pods on disk with userns data. | ||||
| 		podList: pods, | ||||
| 	} | ||||
| 	m, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// Record the pods on disk. | ||||
| 	for _, podUID := range pods { | ||||
| 		pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}} | ||||
| 		_, err := m.GetOrCreateUserNamespaceMappings(&pod, "") | ||||
| 		require.NoError(t, err, "failed to record userns range for pod %v", podUID) | ||||
| 	} | ||||
|  | ||||
| 	// Test re-init works when the feature gate is disabled and there were some | ||||
| 	// pods written on disk. | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, false)() | ||||
| 	m2, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// The feature gate is off, no pods should be allocated. | ||||
| 	for _, pod := range pods { | ||||
| 		ok := m2.podAllocated(pod) | ||||
| 		assert.False(t, ok, "pod %q should not be allocated", pod) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestGetOrCreateUserNamespaceMappingsSwitch(t *testing.T) { | ||||
| 	// Enable the feature gate to create some pods on disk. | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| 	pods := []types.UID{"pod-1", "pod-2"} | ||||
|  | ||||
| 	testUserNsPodsManager := &testUserNsPodsManager{ | ||||
| 		podDir: t.TempDir(), | ||||
| 		// List the same pods we will record, so the second time we create the userns | ||||
| 		// manager, it will find these pods on disk with userns data. | ||||
| 		podList: pods, | ||||
| 	} | ||||
| 	m, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// Record the pods on disk. | ||||
| 	for _, podUID := range pods { | ||||
| 		pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}} | ||||
| 		_, err := m.GetOrCreateUserNamespaceMappings(&pod, "") | ||||
| 		require.NoError(t, err, "failed to record userns range for pod %v", podUID) | ||||
| 	} | ||||
|  | ||||
| 	// Test no-op when the feature gate is disabled and there were some | ||||
| 	// pods registered on disk. | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, false)() | ||||
| 	// Create a new manager with the feature gate off and verify the userns range is nil. | ||||
| 	m2, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	for _, podUID := range pods { | ||||
| 		pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}} | ||||
| 		userns, err := m2.GetOrCreateUserNamespaceMappings(&pod, "") | ||||
|  | ||||
| 		assert.NoError(t, err, "failed to record userns range for pod %v", podUID) | ||||
| 		assert.Nil(t, userns, "userns range should be nil for pod %v", podUID) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestCleanupOrphanedPodUsernsAllocationsSwitch(t *testing.T) { | ||||
| 	// Enable the feature gate to create some pods on disk. | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| 	listPods := []types.UID{"pod-1", "pod-2"} | ||||
| 	pods := []types.UID{"pod-3", "pod-4"} | ||||
| 	testUserNsPodsManager := &testUserNsPodsManager{ | ||||
| 		podDir:  t.TempDir(), | ||||
| 		podList: listPods, | ||||
| 	} | ||||
|  | ||||
| 	m, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// Record the pods on disk. | ||||
| 	for _, podUID := range pods { | ||||
| 		pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}} | ||||
| 		_, err := m.GetOrCreateUserNamespaceMappings(&pod, "") | ||||
| 		require.NoError(t, err, "failed to record userns range for pod %v", podUID) | ||||
| 	} | ||||
|  | ||||
| 	// Test cleanup works when the feature gate is disabled and there were some | ||||
| 	// pods registered. | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, false)() | ||||
| 	err = m.CleanupOrphanedPodUsernsAllocations(nil, nil) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// The feature gate is off, no pods should be allocated. | ||||
| 	for _, pod := range append(listPods, pods...) { | ||||
| 		ok := m.podAllocated(pod) | ||||
| 		assert.False(t, ok, "pod %q should not be allocated", pod) | ||||
| 	} | ||||
| } | ||||
| @@ -34,10 +34,21 @@ import ( | ||||
| 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" | ||||
| ) | ||||
|  | ||||
| const ( | ||||
| 	// skip the first block | ||||
| 	minimumMappingUID = userNsLength | ||||
| 	// allocate enough space for 2000 user namespaces | ||||
| 	mappingLen  = userNsLength * 2000 | ||||
| 	testMaxPods = 110 | ||||
| ) | ||||
|  | ||||
| type testUserNsPodsManager struct { | ||||
| 	podDir  string | ||||
| 	podList []types.UID | ||||
| 	userns  bool | ||||
| 	podDir         string | ||||
| 	podList        []types.UID | ||||
| 	userns         bool | ||||
| 	maxPods        int | ||||
| 	mappingFirstID uint32 | ||||
| 	mappingLen     uint32 | ||||
| } | ||||
|  | ||||
| func (m *testUserNsPodsManager) GetPodDir(podUID types.UID) string { | ||||
| @@ -61,6 +72,21 @@ func (m *testUserNsPodsManager) HandlerSupportsUserNamespaces(runtimeHandler str | ||||
| 	return m.userns, nil | ||||
| } | ||||
|  | ||||
| func (m *testUserNsPodsManager) GetKubeletMappings() (uint32, uint32, error) { | ||||
| 	if m.mappingFirstID != 0 { | ||||
| 		return m.mappingFirstID, m.mappingLen, nil | ||||
| 	} | ||||
| 	return minimumMappingUID, mappingLen, nil | ||||
| } | ||||
|  | ||||
| func (m *testUserNsPodsManager) GetMaxPods() int { | ||||
| 	if m.maxPods != 0 { | ||||
| 		return m.maxPods | ||||
| 	} | ||||
|  | ||||
| 	return testMaxPods | ||||
| } | ||||
|  | ||||
| func TestUserNsManagerAllocate(t *testing.T) { | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| @@ -68,8 +94,6 @@ func TestUserNsManagerAllocate(t *testing.T) { | ||||
| 	m, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	assert.Equal(t, true, m.isSet(0*65536), "m.isSet(0) should be true") | ||||
|  | ||||
| 	allocated, length, err := m.allocateOne("one") | ||||
| 	assert.NoError(t, err) | ||||
| 	assert.Equal(t, userNsLength, int(length), "m.isSet(%d).length=%v", allocated, length) | ||||
| @@ -97,6 +121,9 @@ func TestUserNsManagerAllocate(t *testing.T) { | ||||
| 		allocated, length, err = m.allocateOne(types.UID(fmt.Sprintf("%d", i))) | ||||
| 		assert.Equal(t, userNsLength, int(length), "length is not the expected. iter: %v", i) | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.True(t, allocated >= minimumMappingUID) | ||||
| 		// The last ID of the userns range (allocated+userNsLength) should be within bounds. | ||||
| 		assert.True(t, allocated <= minimumMappingUID+mappingLen-userNsLength) | ||||
| 		allocs = append(allocs, allocated) | ||||
| 	} | ||||
| 	for i, v := range allocs { | ||||
| @@ -111,6 +138,60 @@ func TestUserNsManagerAllocate(t *testing.T) { | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestMakeUserNsManager(t *testing.T) { | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| 	cases := []struct { | ||||
| 		name           string | ||||
| 		mappingFirstID uint32 | ||||
| 		mappingLen     uint32 | ||||
| 		maxPods        int | ||||
| 		success        bool | ||||
| 	}{ | ||||
| 		{ | ||||
| 			name:    "default", | ||||
| 			success: true, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:           "firstID not multiple", | ||||
| 			mappingFirstID: 65536 + 1, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:           "firstID is less than 65535", | ||||
| 			mappingFirstID: 1, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:           "mappingLen not multiple", | ||||
| 			mappingFirstID: 65536, | ||||
| 			mappingLen:     65536 + 1, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name:           "range can't fit maxPods", | ||||
| 			mappingFirstID: 65536, | ||||
| 			mappingLen:     65536, | ||||
| 			maxPods:        2, | ||||
| 		}, | ||||
| 	} | ||||
|  | ||||
| 	for _, tc := range cases { | ||||
| 		t.Run(tc.name, func(t *testing.T) { | ||||
| 			testUserNsPodsManager := &testUserNsPodsManager{ | ||||
| 				podDir:         t.TempDir(), | ||||
| 				mappingFirstID: tc.mappingFirstID, | ||||
| 				mappingLen:     tc.mappingLen, | ||||
| 				maxPods:        tc.maxPods, | ||||
| 			} | ||||
| 			_, err := MakeUserNsManager(testUserNsPodsManager) | ||||
|  | ||||
| 			if tc.success { | ||||
| 				assert.NoError(t, err) | ||||
| 			} else { | ||||
| 				assert.Error(t, err) | ||||
| 			} | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestUserNsManagerParseUserNsFile(t *testing.T) { | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| @@ -366,42 +447,6 @@ func TestCleanupOrphanedPodUsernsAllocations(t *testing.T) { | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestAllocateMaxPods(t *testing.T) { | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| 	testUserNsPodsManager := &testUserNsPodsManager{} | ||||
| 	m, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// The first maxPods allocations should succeed. | ||||
| 	for i := 0; i < maxPods; i++ { | ||||
| 		_, _, err = m.allocateOne(types.UID(fmt.Sprintf("%d", i))) | ||||
| 		require.NoError(t, err) | ||||
| 	} | ||||
|  | ||||
| 	// The next allocation should fail, hitting maxPods. | ||||
| 	_, _, err = m.allocateOne(types.UID(fmt.Sprintf("%d", maxPods+1))) | ||||
| 	assert.Error(t, err) | ||||
| } | ||||
|  | ||||
| func TestRecordMaxPods(t *testing.T) { | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| 	testUserNsPodsManager := &testUserNsPodsManager{} | ||||
| 	m, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// The first maxPods allocations should succeed. | ||||
| 	for i := 0; i < maxPods; i++ { | ||||
| 		err = m.record(types.UID(fmt.Sprintf("%d", i)), uint32((i+1)*65536), 65536) | ||||
| 		require.NoError(t, err) | ||||
| 	} | ||||
|  | ||||
| 	// The next allocation should fail, hitting maxPods. | ||||
| 	err = m.record(types.UID(fmt.Sprintf("%d", maxPods+1)), uint32((maxPods+1)*65536), 65536) | ||||
| 	assert.Error(t, err) | ||||
| } | ||||
|  | ||||
| type failingUserNsPodsManager struct { | ||||
| 	testUserNsPodsManager | ||||
| } | ||||
| @@ -418,3 +463,25 @@ func TestMakeUserNsManagerFailsListPod(t *testing.T) { | ||||
| 	assert.Error(t, err) | ||||
| 	assert.ErrorContains(t, err, "read pods from disk") | ||||
| } | ||||
|  | ||||
| func TestRecordBounds(t *testing.T) { | ||||
| 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)() | ||||
|  | ||||
| 	// Allow exactly for 1 pod | ||||
| 	testUserNsPodsManager := &testUserNsPodsManager{ | ||||
| 		mappingFirstID: 65536, | ||||
| 		mappingLen:     65536, | ||||
| 		maxPods:        1, | ||||
| 	} | ||||
| 	m, err := MakeUserNsManager(testUserNsPodsManager) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// The first pod allocation should succeed. | ||||
| 	err = m.record(types.UID(fmt.Sprintf("%d", 0)), 65536, 65536) | ||||
| 	require.NoError(t, err) | ||||
|  | ||||
| 	// The next allocation should fail, as there is no space left. | ||||
| 	err = m.record(types.UID(fmt.Sprintf("%d", 2)), uint32(2*65536), 65536) | ||||
| 	assert.Error(t, err) | ||||
| 	assert.ErrorContains(t, err, "out of range") | ||||
| } | ||||
|   | ||||
		Reference in New Issue
	
	Block a user
	 Kubernetes Prow Robot
					Kubernetes Prow Robot