e35301c19f
- As discussed in reviews and other public channels, this abstraction is used to represent numa nodes, not sockets. - There is nothing inherently related to sockets in this package anyway.
515 lines
13 KiB
Go
515 lines
13 KiB
Go
/*
|
|
Copyright 2019 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package devicemanager
|
|
|
|
import (
|
|
"reflect"
|
|
"sort"
|
|
"testing"
|
|
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
|
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
|
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
|
|
)
|
|
|
|
type mockAffinityStore struct {
|
|
hint topologymanager.TopologyHint
|
|
}
|
|
|
|
func (m *mockAffinityStore) GetAffinity(podUID string, containerName string) topologymanager.TopologyHint {
|
|
return m.hint
|
|
}
|
|
|
|
func makeNUMADevice(id string, numa int) pluginapi.Device {
|
|
return pluginapi.Device{
|
|
ID: id,
|
|
Topology: &pluginapi.TopologyInfo{Nodes: []*pluginapi.NUMANode{{ID: int64(numa)}}},
|
|
}
|
|
}
|
|
|
|
func topologyHintLessThan(a topologymanager.TopologyHint, b topologymanager.TopologyHint) bool {
|
|
if a.Preferred != b.Preferred {
|
|
return a.Preferred == true
|
|
}
|
|
return a.NUMANodeAffinity.IsNarrowerThan(b.NUMANodeAffinity)
|
|
}
|
|
|
|
func makeSocketMask(sockets ...int) bitmask.BitMask {
|
|
mask, _ := bitmask.NewBitMask(sockets...)
|
|
return mask
|
|
}
|
|
|
|
func TestGetTopologyHints(t *testing.T) {
|
|
tcases := []struct {
|
|
description string
|
|
request map[string]string
|
|
devices map[string][]pluginapi.Device
|
|
allocatedDevices map[string][]string
|
|
expectedHints map[string][]topologymanager.TopologyHint
|
|
}{
|
|
{
|
|
description: "Single Request, no alignment",
|
|
request: map[string]string{
|
|
"testdevice": "1",
|
|
},
|
|
devices: map[string][]pluginapi.Device{
|
|
"testdevice": {
|
|
{ID: "Dev1"},
|
|
{ID: "Dev2"},
|
|
},
|
|
},
|
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
|
"testdevice": nil,
|
|
},
|
|
},
|
|
{
|
|
description: "Single Request, only one with alignment",
|
|
request: map[string]string{
|
|
"testdevice": "1",
|
|
},
|
|
devices: map[string][]pluginapi.Device{
|
|
"testdevice": {
|
|
{ID: "Dev1"},
|
|
makeNUMADevice("Dev2", 1),
|
|
},
|
|
},
|
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
|
"testdevice": {
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
description: "Single Request, one device per socket",
|
|
request: map[string]string{
|
|
"testdevice": "1",
|
|
},
|
|
devices: map[string][]pluginapi.Device{
|
|
"testdevice": {
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
},
|
|
},
|
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
|
"testdevice": {
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
description: "Request for 2, one device per socket",
|
|
request: map[string]string{
|
|
"testdevice": "2",
|
|
},
|
|
devices: map[string][]pluginapi.Device{
|
|
"testdevice": {
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
},
|
|
},
|
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
|
"testdevice": {
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
description: "Request for 2, 2 devices per socket",
|
|
request: map[string]string{
|
|
"testdevice": "2",
|
|
},
|
|
devices: map[string][]pluginapi.Device{
|
|
"testdevice": {
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 0),
|
|
makeNUMADevice("Dev4", 1),
|
|
},
|
|
},
|
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
|
"testdevice": {
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
description: "Request for 2, optimal on 1 NUMA node, forced cross-NUMA",
|
|
request: map[string]string{
|
|
"testdevice": "2",
|
|
},
|
|
devices: map[string][]pluginapi.Device{
|
|
"testdevice": {
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 0),
|
|
makeNUMADevice("Dev4", 1),
|
|
},
|
|
},
|
|
allocatedDevices: map[string][]string{
|
|
"testdevice": {"Dev1", "Dev2"},
|
|
},
|
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
|
"testdevice": {
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
description: "2 device types, mixed configuration",
|
|
request: map[string]string{
|
|
"testdevice1": "2",
|
|
"testdevice2": "1",
|
|
},
|
|
devices: map[string][]pluginapi.Device{
|
|
"testdevice1": {
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 0),
|
|
makeNUMADevice("Dev4", 1),
|
|
},
|
|
"testdevice2": {
|
|
makeNUMADevice("Dev1", 0),
|
|
},
|
|
},
|
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
|
"testdevice1": {
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: false,
|
|
},
|
|
},
|
|
"testdevice2": {
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0),
|
|
Preferred: true,
|
|
},
|
|
{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tcases {
|
|
resourceList := v1.ResourceList{}
|
|
for r := range tc.request {
|
|
resourceList[v1.ResourceName(r)] = resource.MustParse(tc.request[r])
|
|
}
|
|
|
|
pod := makePod(resourceList)
|
|
|
|
m := ManagerImpl{
|
|
allDevices: make(map[string]map[string]pluginapi.Device),
|
|
healthyDevices: make(map[string]sets.String),
|
|
allocatedDevices: make(map[string]sets.String),
|
|
podDevices: make(podDevices),
|
|
sourcesReady: &sourcesReadyStub{},
|
|
activePods: func() []*v1.Pod { return []*v1.Pod{} },
|
|
numaNodes: []int{0, 1},
|
|
}
|
|
|
|
for r := range tc.devices {
|
|
m.allDevices[r] = make(map[string]pluginapi.Device)
|
|
m.healthyDevices[r] = sets.NewString()
|
|
|
|
for _, d := range tc.devices[r] {
|
|
m.allDevices[r][d.ID] = d
|
|
m.healthyDevices[r].Insert(d.ID)
|
|
}
|
|
}
|
|
|
|
for r := range tc.allocatedDevices {
|
|
m.allocatedDevices[r] = sets.NewString()
|
|
|
|
for _, d := range tc.allocatedDevices[r] {
|
|
m.allocatedDevices[r].Insert(d)
|
|
}
|
|
}
|
|
|
|
hints := m.GetTopologyHints(*pod, pod.Spec.Containers[0])
|
|
|
|
for r := range tc.expectedHints {
|
|
sort.SliceStable(hints[r], func(i, j int) bool {
|
|
return topologyHintLessThan(hints[r][i], hints[r][j])
|
|
})
|
|
sort.SliceStable(tc.expectedHints[r], func(i, j int) bool {
|
|
return topologyHintLessThan(tc.expectedHints[r][i], tc.expectedHints[r][j])
|
|
})
|
|
if !reflect.DeepEqual(hints[r], tc.expectedHints[r]) {
|
|
t.Errorf("%v: Expected result to be %v, got %v", tc.description, tc.expectedHints[r], hints[r])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTopologyAlignedAllocation(t *testing.T) {
|
|
tcases := []struct {
|
|
description string
|
|
resource string
|
|
request int
|
|
devices []pluginapi.Device
|
|
allocatedDevices []string
|
|
hint topologymanager.TopologyHint
|
|
expectedAllocation int
|
|
expectedAlignment map[int]int
|
|
}{
|
|
{
|
|
description: "Single Request, no alignment",
|
|
resource: "resource",
|
|
request: 1,
|
|
devices: []pluginapi.Device{
|
|
{ID: "Dev1"},
|
|
{ID: "Dev2"},
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 1,
|
|
expectedAlignment: map[int]int{},
|
|
},
|
|
{
|
|
description: "Request for 1, partial alignment",
|
|
resource: "resource",
|
|
request: 1,
|
|
devices: []pluginapi.Device{
|
|
{ID: "Dev1"},
|
|
makeNUMADevice("Dev2", 1),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 1,
|
|
expectedAlignment: map[int]int{1: 1},
|
|
},
|
|
{
|
|
description: "Single Request, socket 0",
|
|
resource: "resource",
|
|
request: 1,
|
|
devices: []pluginapi.Device{
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(0),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 1,
|
|
expectedAlignment: map[int]int{0: 1},
|
|
},
|
|
{
|
|
description: "Single Request, socket 1",
|
|
resource: "resource",
|
|
request: 1,
|
|
devices: []pluginapi.Device{
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 1,
|
|
expectedAlignment: map[int]int{1: 1},
|
|
},
|
|
{
|
|
description: "Request for 2, socket 0",
|
|
resource: "resource",
|
|
request: 2,
|
|
devices: []pluginapi.Device{
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 0),
|
|
makeNUMADevice("Dev4", 1),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(0),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 2,
|
|
expectedAlignment: map[int]int{0: 2},
|
|
},
|
|
{
|
|
description: "Request for 2, socket 1",
|
|
resource: "resource",
|
|
request: 2,
|
|
devices: []pluginapi.Device{
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 0),
|
|
makeNUMADevice("Dev4", 1),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 2,
|
|
expectedAlignment: map[int]int{1: 2},
|
|
},
|
|
{
|
|
description: "Request for 4, unsatisfiable, prefer socket 0",
|
|
resource: "resource",
|
|
request: 4,
|
|
devices: []pluginapi.Device{
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 0),
|
|
makeNUMADevice("Dev4", 1),
|
|
makeNUMADevice("Dev5", 0),
|
|
makeNUMADevice("Dev6", 1),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(0),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 4,
|
|
expectedAlignment: map[int]int{0: 3, 1: 1},
|
|
},
|
|
{
|
|
description: "Request for 4, unsatisfiable, prefer socket 1",
|
|
resource: "resource",
|
|
request: 4,
|
|
devices: []pluginapi.Device{
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 0),
|
|
makeNUMADevice("Dev4", 1),
|
|
makeNUMADevice("Dev5", 0),
|
|
makeNUMADevice("Dev6", 1),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(1),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 4,
|
|
expectedAlignment: map[int]int{0: 1, 1: 3},
|
|
},
|
|
{
|
|
description: "Request for 4, multisocket",
|
|
resource: "resource",
|
|
request: 4,
|
|
devices: []pluginapi.Device{
|
|
makeNUMADevice("Dev1", 0),
|
|
makeNUMADevice("Dev2", 1),
|
|
makeNUMADevice("Dev3", 2),
|
|
makeNUMADevice("Dev4", 3),
|
|
makeNUMADevice("Dev5", 0),
|
|
makeNUMADevice("Dev6", 1),
|
|
makeNUMADevice("Dev7", 2),
|
|
makeNUMADevice("Dev8", 3),
|
|
},
|
|
hint: topologymanager.TopologyHint{
|
|
NUMANodeAffinity: makeSocketMask(1, 3),
|
|
Preferred: true,
|
|
},
|
|
expectedAllocation: 4,
|
|
expectedAlignment: map[int]int{1: 2, 3: 2},
|
|
},
|
|
}
|
|
for _, tc := range tcases {
|
|
m := ManagerImpl{
|
|
allDevices: make(map[string]map[string]pluginapi.Device),
|
|
healthyDevices: make(map[string]sets.String),
|
|
allocatedDevices: make(map[string]sets.String),
|
|
podDevices: make(podDevices),
|
|
sourcesReady: &sourcesReadyStub{},
|
|
activePods: func() []*v1.Pod { return []*v1.Pod{} },
|
|
topologyAffinityStore: &mockAffinityStore{tc.hint},
|
|
}
|
|
|
|
m.allDevices[tc.resource] = make(map[string]pluginapi.Device)
|
|
m.healthyDevices[tc.resource] = sets.NewString()
|
|
|
|
for _, d := range tc.devices {
|
|
m.allDevices[tc.resource][d.ID] = d
|
|
m.healthyDevices[tc.resource].Insert(d.ID)
|
|
}
|
|
|
|
allocated, err := m.devicesToAllocate("podUID", "containerName", tc.resource, tc.request, sets.NewString())
|
|
if err != nil {
|
|
t.Errorf("Unexpected error: %v", err)
|
|
continue
|
|
}
|
|
|
|
if len(allocated) != tc.expectedAllocation {
|
|
t.Errorf("%v. expected allocation: %v but got: %v", tc.description, tc.expectedAllocation, len(allocated))
|
|
}
|
|
|
|
alignment := make(map[int]int)
|
|
if m.deviceHasTopologyAlignment(tc.resource) {
|
|
for d := range allocated {
|
|
if m.allDevices[tc.resource][d].Topology != nil {
|
|
alignment[int(m.allDevices[tc.resource][d].Topology.Nodes[0].ID)]++
|
|
}
|
|
}
|
|
}
|
|
|
|
if !reflect.DeepEqual(alignment, tc.expectedAlignment) {
|
|
t.Errorf("%v. expected alignment: %v but got: %v", tc.description, tc.expectedAlignment, alignment)
|
|
}
|
|
}
|
|
}
|