diff --git a/pkg/kubelet/apis/podresources/server_v1_test.go b/pkg/kubelet/apis/podresources/server_v1_test.go index 2bc69a00004..de69b2ac1b4 100644 --- a/pkg/kubelet/apis/podresources/server_v1_test.go +++ b/pkg/kubelet/apis/podresources/server_v1_test.go @@ -85,12 +85,14 @@ func TestListPodResourcesV1(t *testing.T) { } pluginCDIDevices := []*podresourcesapi.CDIDevice{{Name: "dra-dev0"}, {Name: "dra-dev1"}} + draDriverName := "dra.example.com" + poolName := "worker-1-pool" + deviceName := "gpu-1" draDevs := []*podresourcesapi.DynamicResource{ { - ClassName: "resource-class", ClaimName: "claim-name", ClaimNamespace: "default", - ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices}}, + ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices, DriverName: draDriverName, PoolName: poolName, DeviceName: deviceName}}, }, } @@ -893,7 +895,6 @@ func TestGetPodResourcesV1(t *testing.T) { pluginCDIDevices := []*podresourcesapi.CDIDevice{{Name: "dra-dev0"}, {Name: "dra-dev1"}} draDevs := []*podresourcesapi.DynamicResource{ { - ClassName: "resource-class", ClaimName: "claim-name", ClaimNamespace: "default", ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices}}, diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 1a884163b3a..c5ec87abfbd 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -661,10 +661,6 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe if err != nil { return nil, err } - // NOTE: Passing CDI device names as annotations is a temporary solution - // It will be removed after all runtimes are updated - // to get CDI device names from the ContainerConfig.CDIDevices field - opts.Annotations = append(opts.Annotations, resOpts.Annotations...) opts.CDIDevices = append(opts.CDIDevices, resOpts.CDIDevices...) } // Allocate should already be called during predicateAdmitHandler.Admit(), @@ -965,19 +961,22 @@ func (cm *containerManagerImpl) GetDynamicResources(pod *v1.Pod, container *v1.C } for _, containerClaimInfo := range containerClaimInfos { var claimResources []*podresourcesapi.ClaimResource - // TODO: Currently we maintain a list of ClaimResources, each of which contains - // a set of CDIDevices from a different kubelet plugin. In the future we may want to - // include the name of the kubelet plugin and/or other types of resources that are - // not CDIDevices (assuming the DRAmanager supports this). - for _, klPluginCdiDevices := range containerClaimInfo.CDIDevices { + for driverName, driverState := range containerClaimInfo.DriverState { var cdiDevices []*podresourcesapi.CDIDevice - for _, cdiDevice := range klPluginCdiDevices { - cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDevice}) + for _, device := range driverState.Devices { + for _, cdiDeviceID := range device.CDIDeviceIDs { + cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDeviceID}) + } + resources := &podresourcesapi.ClaimResource{ + CDIDevices: cdiDevices, + DriverName: driverName, + PoolName: device.PoolName, + DeviceName: device.DeviceName, + } + claimResources = append(claimResources, resources) } - claimResources = append(claimResources, &podresourcesapi.ClaimResource{CDIDevices: cdiDevices}) } containerDynamicResource := podresourcesapi.DynamicResource{ - ClassName: containerClaimInfo.ClassName, ClaimName: containerClaimInfo.ClaimName, ClaimNamespace: containerClaimInfo.Namespace, ClaimResources: claimResources, diff --git a/pkg/kubelet/cm/dra/claiminfo.go b/pkg/kubelet/cm/dra/claiminfo.go index 4ff46d7a976..7cf95d05604 100644 --- a/pkg/kubelet/cm/dra/claiminfo.go +++ b/pkg/kubelet/cm/dra/claiminfo.go @@ -17,14 +17,15 @@ limitations under the License. package dra import ( + "errors" "fmt" + "slices" "sync" resourceapi "k8s.io/api/resource/v1alpha3" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/kubelet/cm/dra/state" - "k8s.io/kubernetes/pkg/kubelet/cm/util/cdi" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -33,10 +34,7 @@ import ( // +k8s:deepcopy-gen=true type ClaimInfo struct { state.ClaimInfoState - // annotations is a mapping of container annotations per DRA plugin associated with - // a prepared resource - annotations map[string][]kubecontainer.Annotation - prepared bool + prepared bool } // claimInfoCache is a cache of processed resource claims keyed by namespace/claimname. @@ -47,89 +45,45 @@ type claimInfoCache struct { } // newClaimInfoFromClaim creates a new claim info from a resource claim. -func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) *ClaimInfo { - // Grab the allocation.resourceHandles. If there are no - // allocation.resourceHandles, create a single resourceHandle with no - // content. This will trigger processing of this claim by a single - // kubelet plugin whose name matches resourceClaim.Status.DriverName. - resourceHandles := claim.Status.Allocation.ResourceHandles - if len(resourceHandles) == 0 { - resourceHandles = make([]resourceapi.ResourceHandle, 1) - } +// It verifies that the kubelet can handle the claim. +func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) (*ClaimInfo, error) { claimInfoState := state.ClaimInfoState{ - DriverName: claim.Status.DriverName, - ClassName: claim.Spec.ResourceClassName, - ClaimUID: claim.UID, - ClaimName: claim.Name, - Namespace: claim.Namespace, - PodUIDs: sets.New[string](), - ResourceHandles: resourceHandles, - CDIDevices: make(map[string][]string), + ClaimUID: claim.UID, + ClaimName: claim.Name, + Namespace: claim.Namespace, + PodUIDs: sets.New[string](), + DriverState: make(map[string]state.DriverState), + } + if claim.Status.Allocation == nil { + return nil, errors.New("not allocated") + } + for _, result := range claim.Status.Allocation.Devices.Results { + claimInfoState.DriverState[result.Driver] = state.DriverState{} } info := &ClaimInfo{ ClaimInfoState: claimInfoState, - annotations: make(map[string][]kubecontainer.Annotation), prepared: false, } - return info + return info, nil } // newClaimInfoFromClaim creates a new claim info from a checkpointed claim info state object. func newClaimInfoFromState(state *state.ClaimInfoState) *ClaimInfo { info := &ClaimInfo{ ClaimInfoState: *state.DeepCopy(), - annotations: make(map[string][]kubecontainer.Annotation), prepared: false, } - for pluginName, devices := range info.CDIDevices { - annotations, _ := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, devices) - info.annotations[pluginName] = append(info.annotations[pluginName], annotations...) - } return info } // setCDIDevices adds a set of CDI devices to the claim info. -func (info *ClaimInfo) setCDIDevices(pluginName string, cdiDevices []string) error { - // NOTE: Passing CDI device names as annotations is a temporary solution - // It will be removed after all runtimes are updated - // to get CDI device names from the ContainerConfig.CDIDevices field - annotations, err := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, cdiDevices) - if err != nil { - return fmt.Errorf("failed to generate container annotations, err: %+v", err) +func (info *ClaimInfo) addDevice(driverName string, device state.Device) { + if info.DriverState == nil { + info.DriverState = make(map[string]state.DriverState) } - - if info.CDIDevices == nil { - info.CDIDevices = make(map[string][]string) - } - - if info.annotations == nil { - info.annotations = make(map[string][]kubecontainer.Annotation) - } - - info.CDIDevices[pluginName] = cdiDevices - info.annotations[pluginName] = annotations - - return nil -} - -// annotationsAsList returns container annotations as a single list. -func (info *ClaimInfo) annotationsAsList() []kubecontainer.Annotation { - var lst []kubecontainer.Annotation - for _, v := range info.annotations { - lst = append(lst, v...) - } - return lst -} - -// cdiDevicesAsList returns a list of CDIDevices from the provided claim info. -func (info *ClaimInfo) cdiDevicesAsList() []kubecontainer.CDIDevice { - var cdiDevices []kubecontainer.CDIDevice - for _, devices := range info.CDIDevices { - for _, device := range devices { - cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: device}) - } - } - return cdiDevices + driverState := info.DriverState[driverName] + driverState.Devices = append(driverState.Devices, device) + info.DriverState[driverName] = driverState } // addPodReference adds a pod reference to the claim info. @@ -240,3 +194,20 @@ func (cache *claimInfoCache) syncToCheckpoint() error { } return cache.state.Store(claimInfoStateList) } + +// cdiDevicesAsList returns a list of CDIDevices from the provided claim info. +// When the request name is non-empty, only devices relevant for that request +// are returned. +func (info *ClaimInfo) cdiDevicesAsList(requestName string) []kubecontainer.CDIDevice { + var cdiDevices []kubecontainer.CDIDevice + for _, driverData := range info.DriverState { + for _, device := range driverData.Devices { + if requestName == "" || len(device.RequestNames) == 0 || slices.Contains(device.RequestNames, requestName) { + for _, cdiDeviceID := range device.CDIDeviceIDs { + cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: cdiDeviceID}) + } + } + } + } + return cdiDevices +} diff --git a/pkg/kubelet/cm/dra/claiminfo_test.go b/pkg/kubelet/cm/dra/claiminfo_test.go index b6bb619d71d..139ecc8c9e2 100644 --- a/pkg/kubelet/cm/dra/claiminfo_test.go +++ b/pkg/kubelet/cm/dra/claiminfo_test.go @@ -18,30 +18,46 @@ package dra import ( "errors" - "fmt" "path" "reflect" - "sort" "testing" "github.com/stretchr/testify/assert" + resourceapi "k8s.io/api/resource/v1alpha3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/kubelet/cm/dra/state" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) // ClaimInfo test cases -func TestNewClaimInfoFromClaim(t *testing.T) { - namespace := "test-namespace" - className := "test-class" - driverName := "test-plugin" - claimUID := types.UID("claim-uid") - claimName := "test-claim" +const ( + namespace = "test-namespace" + className = "test-class" + driverName = "test-driver" + deviceName = "test-device" // name inside ResourceSlice + cdiDeviceName = "cdi-test-device" // name inside CDI spec + cdiID = "test-driver/test=cdi-test-device" // CDI device ID + poolName = "test-pool" + requestName = "test-request" + claimName = "test-claim" + claimUID = types.UID(claimName + "-uid") + podUID = "test-pod-uid" +) +var ( + device = state.Device{ + PoolName: poolName, + DeviceName: deviceName, + RequestNames: []string{requestName}, + CDIDeviceIDs: []string{cdiID}, + } + devices = []state.Device{device} +) + +func TestNewClaimInfoFromClaim(t *testing.T) { for _, test := range []struct { description string claim *resourceapi.ResourceClaim @@ -56,28 +72,41 @@ func TestNewClaimInfoFromClaim(t *testing.T) { Namespace: namespace, }, Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{}, + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + { + Request: requestName, + Pool: poolName, + Device: deviceName, + Driver: driverName, + }, + }, + }, }, }, Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: className, + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{ + { + Name: requestName, + DeviceClassName: className, + }, + }, + }, }, }, expectedResult: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClassName: className, - ClaimUID: claimUID, - ClaimName: claimName, - Namespace: claimName, - PodUIDs: sets.New[string](), - ResourceHandles: []resourceapi.ResourceHandle{ - {}, + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](), + DriverState: map[string]state.DriverState{ + driverName: {}, }, - CDIDevices: make(map[string][]string), }, + prepared: false, }, }, { @@ -89,33 +118,29 @@ func TestNewClaimInfoFromClaim(t *testing.T) { Namespace: namespace, }, Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, Allocation: &resourceapi.AllocationResult{}, }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: className, - }, + Spec: resourceapi.ResourceClaimSpec{}, }, expectedResult: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClassName: className, - ClaimUID: claimUID, - ClaimName: claimName, - Namespace: claimName, - PodUIDs: sets.New[string](), - ResourceHandles: []resourceapi.ResourceHandle{ - {}, - }, - CDIDevices: make(map[string][]string), + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](), + DriverState: map[string]state.DriverState{}, }, + prepared: false, }, }, } { t.Run(test.description, func(t *testing.T) { - result := newClaimInfoFromClaim(test.claim) - if reflect.DeepEqual(result, test.expectedResult) { - t.Errorf("Expected %v, but got %v", test.expectedResult, result) + result, err := newClaimInfoFromClaim(test.claim) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if !reflect.DeepEqual(test.expectedResult, result) { + t.Errorf("Expected %+v, but got %+v", test.expectedResult, result) } }) } @@ -130,237 +155,95 @@ func TestNewClaimInfoFromState(t *testing.T) { { description: "successfully created object", state: &state.ClaimInfoState{ - DriverName: "test-driver", - ClassName: "test-class", - ClaimUID: "test-uid", - ClaimName: "test-claim", - Namespace: "test-namespace", - PodUIDs: sets.New[string]("test-pod-uid"), - ResourceHandles: []resourceapi.ResourceHandle{}, - CDIDevices: map[string][]string{}, + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](podUID), + DriverState: map[string]state.DriverState{ + driverName: { + Devices: devices, + }, + }, + }, + expectedResult: &ClaimInfo{ + ClaimInfoState: state.ClaimInfoState{ + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](podUID), + DriverState: map[string]state.DriverState{ + driverName: { + Devices: devices, + }, + }, + }, + prepared: false, }, }, } { t.Run(test.description, func(t *testing.T) { result := newClaimInfoFromState(test.state) - if reflect.DeepEqual(result, test.expectedResult) { - t.Errorf("Expected %v, but got %v", test.expectedResult, result) + if !reflect.DeepEqual(result, test.expectedResult) { + t.Errorf("Expected %+v, but got %+v", test.expectedResult, result) } }) } } -func TestClaimInfoSetCDIDevices(t *testing.T) { - claimUID := types.UID("claim-uid") - pluginName := "test-plugin" - device := "vendor.com/device=device1" - annotationName := fmt.Sprintf("cdi.k8s.io/%s_%s", pluginName, claimUID) +func TestClaimInfoAddDevice(t *testing.T) { for _, test := range []struct { - description string - claimInfo *ClaimInfo - devices []string - expectedCDIDevices map[string][]string - expectedAnnotations map[string][]kubecontainer.Annotation - wantErr bool + description string + claimInfo *ClaimInfo + device state.Device }{ { - description: "successfully add one device", + description: "add new device", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - DriverName: pluginName, - ClaimUID: claimUID, - }, - }, - devices: []string{device}, - expectedCDIDevices: map[string][]string{ - pluginName: {device}, - }, - expectedAnnotations: map[string][]kubecontainer.Annotation{ - pluginName: { - { - Name: annotationName, - Value: device, - }, + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](podUID), }, + prepared: false, }, + device: device, }, { - description: "empty list of devices", + description: "other new device", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - DriverName: pluginName, - ClaimUID: claimUID, + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](podUID), }, }, - devices: []string{}, - expectedCDIDevices: map[string][]string{pluginName: {}}, - expectedAnnotations: map[string][]kubecontainer.Annotation{pluginName: nil}, - }, - { - description: "incorrect device format", - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: pluginName, - ClaimUID: claimUID, - }, - }, - devices: []string{"incorrect"}, - wantErr: true, + device: func() state.Device { + device := device + device.PoolName += "-2" + device.DeviceName += "-2" + device.RequestNames = []string{device.RequestNames[0] + "-2"} + device.CDIDeviceIDs = []string{device.CDIDeviceIDs[0] + "-2"} + return device + }(), }, } { t.Run(test.description, func(t *testing.T) { - err := test.claimInfo.setCDIDevices(pluginName, test.devices) - if test.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, test.expectedCDIDevices, test.claimInfo.CDIDevices) - assert.Equal(t, test.expectedAnnotations, test.claimInfo.annotations) + test.claimInfo.addDevice(driverName, test.device) + assert.Equal(t, []state.Device{test.device}, test.claimInfo.DriverState[driverName].Devices) }) } } -func TestClaimInfoAnnotationsAsList(t *testing.T) { - for _, test := range []struct { - description string - claimInfo *ClaimInfo - expectedResult []kubecontainer.Annotation - }{ - { - description: "empty annotations", - claimInfo: &ClaimInfo{ - annotations: map[string][]kubecontainer.Annotation{}, - }, - }, - { - description: "nil annotations", - claimInfo: &ClaimInfo{}, - }, - { - description: "valid annotations", - claimInfo: &ClaimInfo{ - annotations: map[string][]kubecontainer.Annotation{ - "test-plugin1": { - { - Name: "cdi.k8s.io/test-plugin1_claim-uid1", - Value: "vendor.com/device=device1", - }, - { - Name: "cdi.k8s.io/test-plugin1_claim-uid2", - Value: "vendor.com/device=device2", - }, - }, - "test-plugin2": { - { - Name: "cdi.k8s.io/test-plugin2_claim-uid1", - Value: "vendor.com/device=device1", - }, - { - Name: "cdi.k8s.io/test-plugin2_claim-uid2", - Value: "vendor.com/device=device2", - }, - }, - }, - }, - expectedResult: []kubecontainer.Annotation{ - { - Name: "cdi.k8s.io/test-plugin1_claim-uid1", - Value: "vendor.com/device=device1", - }, - { - Name: "cdi.k8s.io/test-plugin1_claim-uid2", - Value: "vendor.com/device=device2", - }, - { - Name: "cdi.k8s.io/test-plugin2_claim-uid1", - Value: "vendor.com/device=device1", - }, - { - Name: "cdi.k8s.io/test-plugin2_claim-uid2", - Value: "vendor.com/device=device2", - }, - }, - }, - } { - t.Run(test.description, func(t *testing.T) { - result := test.claimInfo.annotationsAsList() - sort.Slice(result, func(i, j int) bool { - return result[i].Name < result[j].Name - }) - assert.Equal(t, test.expectedResult, result) - }) - } -} - -func TestClaimInfoCDIdevicesAsList(t *testing.T) { - for _, test := range []struct { - description string - claimInfo *ClaimInfo - expectedResult []kubecontainer.CDIDevice - }{ - { - description: "empty CDI devices", - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - CDIDevices: map[string][]string{}, - }, - }, - }, - { - description: "nil CDI devices", - claimInfo: &ClaimInfo{}, - }, - { - description: "valid CDI devices", - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - CDIDevices: map[string][]string{ - "test-plugin1": { - "vendor.com/device=device1", - "vendor.com/device=device2", - }, - "test-plugin2": { - "vendor.com/device=device1", - "vendor.com/device=device2", - }, - }, - }, - }, - expectedResult: []kubecontainer.CDIDevice{ - { - Name: "vendor.com/device=device1", - }, - { - Name: "vendor.com/device=device1", - }, - { - Name: "vendor.com/device=device2", - }, - { - Name: "vendor.com/device=device2", - }, - }, - }, - } { - t.Run(test.description, func(t *testing.T) { - result := test.claimInfo.cdiDevicesAsList() - sort.Slice(result, func(i, j int) bool { - return result[i].Name < result[j].Name - }) - assert.Equal(t, test.expectedResult, result) - }) - } -} func TestClaimInfoAddPodReference(t *testing.T) { - podUID := types.UID("pod-uid") for _, test := range []struct { description string claimInfo *ClaimInfo expectedLen int }{ { - description: "successfully add pod reference", + description: "empty PodUIDs list", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ PodUIDs: sets.New[string](), @@ -369,16 +252,16 @@ func TestClaimInfoAddPodReference(t *testing.T) { expectedLen: 1, }, { - description: "duplicate pod reference", + description: "first pod reference", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - PodUIDs: sets.New[string](string(podUID)), + PodUIDs: sets.New[string](podUID), }, }, expectedLen: 1, }, { - description: "duplicate pod reference", + description: "second pod reference", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ PodUIDs: sets.New[string]("pod-uid1"), @@ -396,7 +279,6 @@ func TestClaimInfoAddPodReference(t *testing.T) { } func TestClaimInfoHasPodReference(t *testing.T) { - podUID := types.UID("pod-uid") for _, test := range []struct { description string claimInfo *ClaimInfo @@ -414,7 +296,7 @@ func TestClaimInfoHasPodReference(t *testing.T) { description: "claim references pod", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - PodUIDs: sets.New[string](string(podUID)), + PodUIDs: sets.New[string](podUID), }, }, expectedResult: true, @@ -431,7 +313,6 @@ func TestClaimInfoHasPodReference(t *testing.T) { } func TestClaimInfoDeletePodReference(t *testing.T) { - podUID := types.UID("pod-uid") for _, test := range []struct { description string claimInfo *ClaimInfo @@ -448,7 +329,7 @@ func TestClaimInfoDeletePodReference(t *testing.T) { description: "claim references pod", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - PodUIDs: sets.New[string](string(podUID)), + PodUIDs: sets.New[string](podUID), }, }, }, @@ -694,8 +575,8 @@ func TestClaimInfoCacheAdd(t *testing.T) { description: "claimInfo successfully added", claimInfo: &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{ - ClaimName: "test-claim", - Namespace: "test-namespace", + ClaimName: claimName, + Namespace: namespace, }, }, }, @@ -711,8 +592,6 @@ func TestClaimInfoCacheAdd(t *testing.T) { } func TestClaimInfoCacheContains(t *testing.T) { - claimName := "test-claim" - namespace := "test-namespace" for _, test := range []struct { description string claimInfo *ClaimInfo @@ -764,8 +643,6 @@ func TestClaimInfoCacheContains(t *testing.T) { } func TestClaimInfoCacheGet(t *testing.T) { - claimName := "test-claim" - namespace := "test-namespace" for _, test := range []struct { description string claimInfoCache *claimInfoCache @@ -801,8 +678,6 @@ func TestClaimInfoCacheGet(t *testing.T) { } func TestClaimInfoCacheDelete(t *testing.T) { - claimName := "test-claim" - namespace := "test-namespace" for _, test := range []struct { description string claimInfoCache *claimInfoCache @@ -833,9 +708,6 @@ func TestClaimInfoCacheDelete(t *testing.T) { } func TestClaimInfoCacheHasPodReference(t *testing.T) { - claimName := "test-claim" - namespace := "test-namespace" - uid := types.UID("test-uid") for _, test := range []struct { description string claimInfoCache *claimInfoCache @@ -849,7 +721,7 @@ func TestClaimInfoCacheHasPodReference(t *testing.T) { ClaimInfoState: state.ClaimInfoState{ ClaimName: claimName, Namespace: namespace, - PodUIDs: sets.New[string](string(uid)), + PodUIDs: sets.New[string](podUID), }, }, }, @@ -862,7 +734,7 @@ func TestClaimInfoCacheHasPodReference(t *testing.T) { }, } { t.Run(test.description, func(t *testing.T) { - assert.Equal(t, test.expectedResult, test.claimInfoCache.hasPodReference(uid)) + assert.Equal(t, test.expectedResult, test.claimInfoCache.hasPodReference(podUID)) }) } } diff --git a/pkg/kubelet/cm/dra/manager.go b/pkg/kubelet/cm/dra/manager.go index c226bb35b22..bd8416003fc 100644 --- a/pkg/kubelet/cm/dra/manager.go +++ b/pkg/kubelet/cm/dra/manager.go @@ -32,6 +32,7 @@ import ( "k8s.io/klog/v2" drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4" dra "k8s.io/kubernetes/pkg/kubelet/cm/dra/plugin" + "k8s.io/kubernetes/pkg/kubelet/cm/dra/state" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -45,7 +46,7 @@ const defaultReconcilePeriod = 60 * time.Second // ActivePodsFunc is a function that returns a list of pods to reconcile. type ActivePodsFunc func() []*v1.Pod -// ManagerImpl is the structure in charge of managing DRA resource Plugins. +// ManagerImpl is the structure in charge of managing DRA drivers. type ManagerImpl struct { // cache contains cached claim info cache *claimInfoCache @@ -145,8 +146,8 @@ func (m *ManagerImpl) reconcileLoop() { } } -// PrepareResources attempts to prepare all of the required resource -// plugin resources for the input container, issue NodePrepareResources rpc requests +// PrepareResources attempts to prepare all of the required resources +// for the input container, issue NodePrepareResources rpc requests // for each new resource requirement, process their responses and update the cached // containerResources on success. func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { @@ -154,7 +155,7 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { resourceClaims := make(map[types.UID]*resourceapi.ResourceClaim) for i := range pod.Spec.ResourceClaims { podClaim := &pod.Spec.ResourceClaims[i] - klog.V(3).InfoS("Processing resource", "podClaim", podClaim.Name, "pod", pod.Name) + klog.V(3).InfoS("Processing resource", "pod", klog.KObj(pod), "podClaim", podClaim.Name) claimName, mustCheckOwner, err := resourceclaim.Name(pod, podClaim) if err != nil { return fmt.Errorf("prepare resource claim: %v", err) @@ -162,6 +163,7 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { if claimName == nil { // Nothing to do. + klog.V(5).InfoS("No need to prepare resources, no claim generated", "pod", klog.KObj(pod), "podClaim", podClaim.Name) continue } // Query claim object from the API server @@ -185,20 +187,20 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { pod.Name, pod.UID, *claimName, resourceClaim.UID) } - // If no container actually uses the claim, then we don't need - // to prepare it. - if !claimIsUsedByPod(podClaim, pod) { - klog.V(5).InfoS("Skipping unused resource", "claim", claimName, "pod", pod.Name) - continue - } - // Atomically perform some operations on the claimInfo cache. err = m.cache.withLock(func() error { // Get a reference to the claim info for this claim from the cache. // If there isn't one yet, then add it to the cache. claimInfo, exists := m.cache.get(resourceClaim.Name, resourceClaim.Namespace) if !exists { - claimInfo = m.cache.add(newClaimInfoFromClaim(resourceClaim)) + ci, err := newClaimInfoFromClaim(resourceClaim) + if err != nil { + return fmt.Errorf("claim %s: %w", klog.KObj(resourceClaim), err) + } + claimInfo = m.cache.add(ci) + klog.V(6).InfoS("Created new claim info cache entry", "pod", klog.KObj(pod), "podClaim", podClaim.Name, "claim", klog.KObj(resourceClaim), "claimInfoEntry", claimInfo) + } else { + klog.V(6).InfoS("Found existing claim info cache entry", "pod", klog.KObj(pod), "podClaim", podClaim.Name, "claim", klog.KObj(resourceClaim), "claimInfoEntry", claimInfo) } // Add a reference to the current pod in the claim info. @@ -214,6 +216,7 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { // If this claim is already prepared, there is no need to prepare it again. if claimInfo.isPrepared() { + klog.V(5).InfoS("Resources already prepared", "pod", klog.KObj(pod), "podClaim", podClaim.Name, "claim", klog.KObj(resourceClaim)) return nil } @@ -221,15 +224,14 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { // after NodePrepareResources GRPC succeeds resourceClaims[claimInfo.ClaimUID] = resourceClaim - // Loop through all plugins and prepare for calling NodePrepareResources. - for _, resourceHandle := range claimInfo.ResourceHandles { - claim := &drapb.Claim{ - Namespace: claimInfo.Namespace, - Uid: string(claimInfo.ClaimUID), - Name: claimInfo.ClaimName, - } - pluginName := resourceHandle.DriverName - batches[pluginName] = append(batches[pluginName], claim) + // Loop through all drivers and prepare for calling NodePrepareResources. + claim := &drapb.Claim{ + Namespace: claimInfo.Namespace, + UID: string(claimInfo.ClaimUID), + Name: claimInfo.ClaimName, + } + for driverName := range claimInfo.DriverState { + batches[driverName] = append(batches[driverName], claim) } return nil @@ -242,16 +244,16 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { // Call NodePrepareResources for all claims in each batch. // If there is any error, processing gets aborted. // We could try to continue, but that would make the code more complex. - for pluginName, claims := range batches { + for driverName, claims := range batches { // Call NodePrepareResources RPC for all resource handles. - client, err := dra.NewDRAPluginClient(pluginName) + client, err := dra.NewDRAPluginClient(driverName) if err != nil { - return fmt.Errorf("failed to get DRA Plugin client for plugin name %s: %v", pluginName, err) + return fmt.Errorf("failed to get gRPC client for driver %s: %w", driverName, err) } response, err := client.NodePrepareResources(context.Background(), &drapb.NodePrepareResourcesRequest{Claims: claims}) if err != nil { // General error unrelated to any particular claim. - return fmt.Errorf("NodePrepareResources failed: %v", err) + return fmt.Errorf("NodePrepareResources failed: %w", err) } for claimUID, result := range response.Claims { reqClaim := lookupClaimRequest(claims, claimUID) @@ -270,8 +272,8 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { if !exists { return fmt.Errorf("unable to get claim info for claim %s in namespace %s", claim.Name, claim.Namespace) } - if err := info.setCDIDevices(pluginName, result.GetCDIDevices()); err != nil { - return fmt.Errorf("unable to add CDI devices for plugin %s of claim %s in namespace %s", pluginName, claim.Name, claim.Namespace) + for _, device := range result.GetDevices() { + info.addDevice(driverName, state.Device{PoolName: device.PoolName, DeviceName: device.DeviceName, RequestNames: device.RequestNames, CDIDeviceIDs: device.CDIDeviceIDs}) } return nil }) @@ -314,60 +316,33 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { func lookupClaimRequest(claims []*drapb.Claim, claimUID string) *drapb.Claim { for _, claim := range claims { - if claim.Uid == claimUID { + if claim.UID == claimUID { return claim } } return nil } -func claimIsUsedByPod(podClaim *v1.PodResourceClaim, pod *v1.Pod) bool { - if claimIsUsedByContainers(podClaim, pod.Spec.InitContainers) { - return true - } - if claimIsUsedByContainers(podClaim, pod.Spec.Containers) { - return true - } - return false -} - -func claimIsUsedByContainers(podClaim *v1.PodResourceClaim, containers []v1.Container) bool { - for i := range containers { - if claimIsUsedByContainer(podClaim, &containers[i]) { - return true - } - } - return false -} - -func claimIsUsedByContainer(podClaim *v1.PodResourceClaim, container *v1.Container) bool { - for _, c := range container.Resources.Claims { - if c.Name == podClaim.Name { - return true - } - } - return false -} - // GetResources gets a ContainerInfo object from the claimInfo cache. // This information is used by the caller to update a container config. func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*ContainerInfo, error) { - annotations := []kubecontainer.Annotation{} cdiDevices := []kubecontainer.CDIDevice{} - for i, podResourceClaim := range pod.Spec.ResourceClaims { - claimName, _, err := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i]) + for i := range pod.Spec.ResourceClaims { + podClaim := &pod.Spec.ResourceClaims[i] + claimName, _, err := resourceclaim.Name(pod, podClaim) if err != nil { - return nil, fmt.Errorf("list resource claims: %v", err) + return nil, fmt.Errorf("list resource claims: %w", err) } // The claim name might be nil if no underlying resource claim // was generated for the referenced claim. There are valid use // cases when this might happen, so we simply skip it. if claimName == nil { + klog.V(5).InfoS("No CDI devices, no claim generated", "pod", klog.KObj(pod), "podClaimName", podClaim.Name) continue } for _, claim := range container.Resources.Claims { - if podResourceClaim.Name != claim.Name { + if podClaim.Name != claim.Name { continue } @@ -377,13 +352,8 @@ func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*Conta return fmt.Errorf("unable to get claim info for claim %s in namespace %s", *claimName, pod.Namespace) } - claimAnnotations := claimInfo.annotationsAsList() - klog.V(3).InfoS("Add resource annotations", "claim", *claimName, "annotations", claimAnnotations) - annotations = append(annotations, claimAnnotations...) - - devices := claimInfo.cdiDevicesAsList() - klog.V(3).InfoS("Add CDI devices", "claim", *claimName, "CDI devices", devices) - cdiDevices = append(cdiDevices, devices...) + // As of Kubernetes 1.31, CDI device IDs are not passed via annotations anymore. + cdiDevices = append(cdiDevices, claimInfo.cdiDevicesAsList(claim.Request)...) return nil }) @@ -393,10 +363,11 @@ func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*Conta } } - return &ContainerInfo{Annotations: annotations, CDIDevices: cdiDevices}, nil + klog.V(5).InfoS("Determined CDI devices for pod", "pod", klog.KObj(pod), "cdiDevices", cdiDevices) + return &ContainerInfo{CDIDevices: cdiDevices}, nil } -// UnprepareResources calls a plugin's NodeUnprepareResource API for each resource claim owned by a pod. +// UnprepareResources calls a driver's NodeUnprepareResource API for each resource claim owned by a pod. // This function is idempotent and may be called multiple times against the same pod. // As such, calls to the underlying NodeUnprepareResource API are skipped for claims that have // already been successfully unprepared. @@ -405,7 +376,7 @@ func (m *ManagerImpl) UnprepareResources(pod *v1.Pod) error { for i := range pod.Spec.ResourceClaims { claimName, _, err := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i]) if err != nil { - return fmt.Errorf("unprepare resource claim: %v", err) + return fmt.Errorf("unprepare resource claim: %w", err) } // The claim name might be nil if no underlying resource claim // was generated for the referenced claim. There are valid use @@ -448,15 +419,14 @@ func (m *ManagerImpl) unprepareResources(podUID types.UID, namespace string, cla // after NodeUnprepareResources GRPC succeeds claimNamesMap[claimInfo.ClaimUID] = claimInfo.ClaimName - // Loop through all plugins and prepare for calling NodeUnprepareResources. - for _, resourceHandle := range claimInfo.ResourceHandles { - claim := &drapb.Claim{ - Namespace: claimInfo.Namespace, - Uid: string(claimInfo.ClaimUID), - Name: claimInfo.ClaimName, - } - pluginName := resourceHandle.DriverName - batches[pluginName] = append(batches[pluginName], claim) + // Loop through all drivers and prepare for calling NodeUnprepareResources. + claim := &drapb.Claim{ + Namespace: claimInfo.Namespace, + UID: string(claimInfo.ClaimUID), + Name: claimInfo.ClaimName, + } + for driverName := range claimInfo.DriverState { + batches[driverName] = append(batches[driverName], claim) } return nil @@ -469,16 +439,16 @@ func (m *ManagerImpl) unprepareResources(podUID types.UID, namespace string, cla // Call NodeUnprepareResources for all claims in each batch. // If there is any error, processing gets aborted. // We could try to continue, but that would make the code more complex. - for pluginName, claims := range batches { + for driverName, claims := range batches { // Call NodeUnprepareResources RPC for all resource handles. - client, err := dra.NewDRAPluginClient(pluginName) + client, err := dra.NewDRAPluginClient(driverName) if err != nil { - return fmt.Errorf("failed to get DRA Plugin client for plugin name %s: %v", pluginName, err) + return fmt.Errorf("get gRPC client for DRA driver %s: %w", driverName, err) } response, err := client.NodeUnprepareResources(context.Background(), &drapb.NodeUnprepareResourcesRequest{Claims: claims}) if err != nil { // General error unrelated to any particular claim. - return fmt.Errorf("NodeUnprepareResources failed: %v", err) + return fmt.Errorf("NodeUnprepareResources failed: %w", err) } for claimUID, result := range response.Claims { @@ -501,7 +471,9 @@ func (m *ManagerImpl) unprepareResources(podUID types.UID, namespace string, cla err := m.cache.withLock(func() error { // Delete all claimInfos from the cache that have just been unprepared. for _, claimName := range claimNamesMap { + claimInfo, _ := m.cache.get(claimName, namespace) m.cache.delete(claimName, namespace) + klog.V(6).InfoS("Deleted claim info cache entry", "claim", klog.KRef(namespace, claimName), "claimInfoEntry", claimInfo) } // Atomically sync the cache back to the checkpoint. @@ -532,7 +504,7 @@ func (m *ManagerImpl) GetContainerClaimInfos(pod *v1.Pod, container *v1.Containe for i, podResourceClaim := range pod.Spec.ResourceClaims { claimName, _, err := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i]) if err != nil { - return nil, fmt.Errorf("determine resource claim information: %v", err) + return nil, fmt.Errorf("determine resource claim information: %w", err) } for _, claim := range container.Resources.Claims { diff --git a/pkg/kubelet/cm/dra/manager_test.go b/pkg/kubelet/cm/dra/manager_test.go index 848ef76b6ba..dc8f26a741a 100644 --- a/pkg/kubelet/cm/dra/manager_test.go +++ b/pkg/kubelet/cm/dra/manager_test.go @@ -28,7 +28,9 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/grpc" + v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,12 +41,12 @@ import ( drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4" "k8s.io/kubernetes/pkg/kubelet/cm/dra/plugin" "k8s.io/kubernetes/pkg/kubelet/cm/dra/state" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) const ( - driverName = "test-cdi-device" driverClassName = "test" + podName = "test-pod" + containerName = "test-container" ) type fakeDRADriverGRPCServer struct { @@ -65,12 +67,19 @@ func (s *fakeDRADriverGRPCServer) NodePrepareResources(ctx context.Context, req } if s.prepareResourcesResponse == nil { - deviceName := "claim-" + req.Claims[0].Uid - result := s.driverName + "/" + driverClassName + "=" + deviceName + cdiDeviceName := "claim-" + req.Claims[0].UID + cdiID := s.driverName + "/" + driverClassName + "=" + cdiDeviceName return &drapb.NodePrepareResourcesResponse{ Claims: map[string]*drapb.NodePrepareResourceResponse{ - req.Claims[0].Uid: { - CDIDevices: []string{result}, + req.Claims[0].UID: { + Devices: []*drapb.Device{ + { + PoolName: poolName, + DeviceName: deviceName, + RequestNames: []string{req.Claims[0].Name}, + CDIDeviceIDs: []string{cdiID}, + }, + }, }, }, }, nil @@ -89,7 +98,7 @@ func (s *fakeDRADriverGRPCServer) NodeUnprepareResources(ctx context.Context, re if s.unprepareResourcesResponse == nil { return &drapb.NodeUnprepareResourcesResponse{ Claims: map[string]*drapb.NodeUnprepareResourceResponse{ - req.Claims[0].Uid: {}, + req.Claims[0].UID: {}, }, }, nil } @@ -193,9 +202,117 @@ func TestNewManagerImpl(t *testing.T) { } } +// genTestPod generates pod object +func genTestPod() *v1.Pod { + claimName := claimName + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + UID: podUID, + }, + Spec: v1.PodSpec{ + ResourceClaims: []v1.PodResourceClaim{ + { + Name: claimName, + ResourceClaimName: &claimName, + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Claims: []v1.ResourceClaim{ + { + Name: claimName, + }, + }, + }, + }, + }, + }, + } +} + +// getTestClaim generates resource claim object +func genTestClaim(name, driver, device, podUID string) *resourceapi.ResourceClaim { + return &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID(fmt.Sprintf("%s-uid", name)), + }, + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{ + { + Name: requestName, + DeviceClassName: className, + }, + }, + }, + }, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + { + Request: requestName, + Pool: poolName, + Device: device, + Driver: driver, + }, + }, + }, + }, + ReservedFor: []resourceapi.ResourceClaimConsumerReference{ + {UID: types.UID(podUID)}, + }, + }, + } +} + +// genTestClaimInfo generates claim info object +func genTestClaimInfo(podUIDs []string, prepared bool) *ClaimInfo { + return &ClaimInfo{ + ClaimInfoState: state.ClaimInfoState{ + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](podUIDs...), + DriverState: map[string]state.DriverState{ + driverName: { + Devices: []state.Device{{ + PoolName: poolName, + DeviceName: deviceName, + RequestNames: []string{requestName}, + CDIDeviceIDs: []string{cdiID}, + }}, + }, + }, + }, + prepared: prepared, + } +} + +// genClaimInfoState generates claim info state object +func genClaimInfoState(cdiDeviceID string) state.ClaimInfoState { + s := state.ClaimInfoState{ + ClaimUID: claimUID, + ClaimName: claimName, + Namespace: namespace, + PodUIDs: sets.New[string](podUID), + DriverState: map[string]state.DriverState{ + driverName: {}, + }, + } + if cdiDeviceID != "" { + s.DriverState[driverName] = state.DriverState{Devices: []state.Device{{PoolName: poolName, DeviceName: deviceName, RequestNames: []string{requestName}, CDIDeviceIDs: []string{cdiDeviceID}}}} + } + return s +} + func TestGetResources(t *testing.T) { kubeClient := fake.NewSimpleClientset() - resourceClaimName := "test-pod-claim-1" for _, test := range []struct { description string @@ -205,110 +322,33 @@ func TestGetResources(t *testing.T) { wantErr bool }{ { - description: "claim info with annotations", + description: "claim info with devices", container: &v1.Container{ - Name: "test-container", + Name: containerName, Resources: v1.ResourceRequirements{ Claims: []v1.ResourceClaim{ { - Name: "test-pod-claim-1", + Name: claimName, }, }, }, }, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-1", - ResourceClaimName: &resourceClaimName, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - annotations: map[string][]kubecontainer.Annotation{ - "test-plugin": { - { - Name: "test-annotation", - Value: "123", - }, - }, - }, - ClaimInfoState: state.ClaimInfoState{ - ClaimName: "test-pod-claim-1", - CDIDevices: map[string][]string{ - driverName: {"123"}, - }, - Namespace: "test-namespace", - }, - }, + pod: genTestPod(), + claimInfo: genTestClaimInfo(nil, false), }, { - description: "claim info without annotations", + description: "nil claiminfo", container: &v1.Container{ - Name: "test-container", + Name: containerName, Resources: v1.ResourceRequirements{ Claims: []v1.ResourceClaim{ { - Name: "test-pod-claim-1", - }, - }, - }, - }, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-1", - ResourceClaimName: &resourceClaimName, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - ClaimName: "test-pod-claim-1", - CDIDevices: map[string][]string{ - driverName: {"123"}, - }, - Namespace: "test-namespace", - }, - }, - }, - { - description: "no claim info", - container: &v1.Container{ - Name: "test-container", - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-1", - }, - }, - }, - }, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-1", + Name: claimName, }, }, }, }, + pod: genTestPod(), wantErr: true, }, } { @@ -323,11 +363,10 @@ func TestGetResources(t *testing.T) { containerInfo, err := manager.GetResources(test.pod, test.container) if test.wantErr { assert.Error(t, err) - return + } else { + require.NoError(t, err) + assert.Equal(t, test.claimInfo.DriverState[driverName].Devices[0].CDIDeviceIDs[0], containerInfo.CDIDevices[0].Name) } - - assert.NoError(t, err) - assert.Equal(t, test.claimInfo.CDIDevices[driverName][0], containerInfo.CDIDevices[0].Name) }) } } @@ -337,560 +376,165 @@ func getFakeNode() (*v1.Node, error) { } func TestPrepareResources(t *testing.T) { + claimName := claimName fakeKubeClient := fake.NewSimpleClientset() for _, test := range []struct { - description string - driverName string - pod *v1.Pod - claimInfo *ClaimInfo - resourceClaim *resourceapi.ResourceClaim - resp *drapb.NodePrepareResourcesResponse - wantErr bool - wantTimeout bool - wantResourceSkipped bool - expectedCDIDevices []string - ExpectedPrepareCalls uint32 + description string + driverName string + pod *v1.Pod + claimInfo *ClaimInfo + claim *resourceapi.ResourceClaim + resp *drapb.NodePrepareResourcesResponse + wantTimeout bool + wantResourceSkipped bool + + expectedErrMsg string + expectedClaimInfoState state.ClaimInfoState + expectedPrepareCalls uint32 }{ { - description: "failed to fetch ResourceClaim", + description: "claim doesn't exist", + driverName: driverName, + pod: genTestPod(), + expectedErrMsg: "failed to fetch ResourceClaim ", + }, + { + description: "unknown driver", + pod: genTestPod(), + claim: genTestClaim(claimName, "unknown driver", deviceName, podUID), + expectedErrMsg: "plugin name unknown driver not found in the list of registered DRA plugins", + }, + { + description: "should prepare resources, driver returns nil value", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), + resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{string(claimUID): nil}}, + expectedClaimInfoState: genClaimInfoState(""), + expectedPrepareCalls: 1, + }, + { + description: "driver returns empty result", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), + resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{}}, + expectedPrepareCalls: 1, + expectedErrMsg: "NodePrepareResources left out 1 claims", + }, + { + description: "pod is not allowed to use resource claim", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, ""), + expectedErrMsg: "is not allowed to use resource claim ", + }, + { + description: "no container uses the claim", driverName: driverName, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", + Name: podName, + Namespace: namespace, + UID: podUID, }, Spec: v1.PodSpec{ ResourceClaims: []v1.PodResourceClaim{ { - Name: "test-pod-claim-0", - - ResourceClaimName: func() *string { - s := "test-pod-claim-0" - return &s - }(), - }, - }, - }, - }, - wantErr: true, - }, - { - description: "plugin does not exist", - driverName: "this-plugin-does-not-exist", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-1", - ResourceClaimName: func() *string { - s := "test-pod-claim-1" - return &s - }(), + Name: claimName, + ResourceClaimName: &claimName, }, }, Containers: []v1.Container{ + {}, + }, + }, + }, + claim: genTestClaim(claimName, driverName, deviceName, podUID), + expectedPrepareCalls: 1, + expectedClaimInfoState: genClaimInfoState(cdiID), + resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{ + string(claimUID): { + Devices: []*drapb.Device{ { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-1", - }, - }, - }, + PoolName: poolName, + DeviceName: deviceName, + RequestNames: []string{requestName}, + CDIDeviceIDs: []string{cdiID}, }, }, }, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-1", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - wantErr: true, + }}, }, { - description: "should prepare resources, driver returns nil value", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ + description: "resource already prepared", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), + claimInfo: genTestClaimInfo([]string{podUID}, true), + expectedClaimInfoState: genClaimInfoState(cdiID), + resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{ + string(claimUID): { + Devices: []*drapb.Device{ { - Name: "test-pod-claim-nil", - ResourceClaimName: func() *string { - s := "test-pod-claim-nil" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-nil", - }, - }, - }, + PoolName: poolName, + DeviceName: deviceName, + RequestNames: []string{requestName}, + CDIDeviceIDs: []string{cdiID}, }, }, }, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-nil", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"test-reserved": nil}}, - expectedCDIDevices: []string{}, - ExpectedPrepareCalls: 1, + }}, }, { - description: "should prepare resources, driver returns empty result", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-empty", - ResourceClaimName: func() *string { - s := "test-pod-claim-empty" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-empty", - }, - }, - }, - }, - }, - }, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-empty", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"test-reserved": nil}}, - expectedCDIDevices: []string{}, - ExpectedPrepareCalls: 1, - }, - { - description: "pod is not allowed to use resource claim", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-2", - ResourceClaimName: func() *string { - s := "test-pod-claim-2" - return &s - }(), - }, - }, - }, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-2", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - }, - }, - wantErr: true, - }, - { - description: "no container actually uses the claim", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-3", - ResourceClaimName: func() *string { - s := "test-pod-claim-3" - return &s - }(), - }, - }, - }, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-3", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - wantResourceSkipped: true, - }, - { - description: "resource already prepared", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-4", - ResourceClaimName: func() *string { - s := "test-pod-claim-4" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-4", - }, - }, - }, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClaimName: "test-pod-claim-4", - Namespace: "test-namespace", - PodUIDs: sets.Set[string]{"test-another-pod-reserved": sets.Empty{}}, - }, - prepared: true, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-4", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - expectedCDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}, - wantResourceSkipped: true, - }, - { - description: "should timeout", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-5", - ResourceClaimName: func() *string { - s := "test-pod-claim-5" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-5", - }, - }, - }, - }, - }, - }, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-5", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - resp: &drapb.NodePrepareResourcesResponse{ - Claims: map[string]*drapb.NodePrepareResourceResponse{ - "test-reserved": {CDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}}, - }, - }, - wantErr: true, + description: "should timeout", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), wantTimeout: true, - ExpectedPrepareCalls: 1, + expectedPrepareCalls: 1, + expectedErrMsg: "NodePrepareResources failed: rpc error: code = DeadlineExceeded desc = context deadline exceeded", }, { - description: "should prepare resource, claim not in cache", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ + description: "should prepare resource, claim not in cache", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), + expectedClaimInfoState: genClaimInfoState(cdiID), + resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{ + string(claimUID): { + Devices: []*drapb.Device{ { - Name: "test-pod-claim-6", - ResourceClaimName: func() *string { - s := "test-pod-claim-6" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-6", - }, - }, - }, + PoolName: poolName, + DeviceName: deviceName, + RequestNames: []string{requestName}, + CDIDeviceIDs: []string{cdiID}, }, }, }, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim-6", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - resp: &drapb.NodePrepareResourcesResponse{ - Claims: map[string]*drapb.NodePrepareResourceResponse{ - "test-reserved": {CDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}}, - }, - }, - expectedCDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}, - ExpectedPrepareCalls: 1, + }}, + expectedPrepareCalls: 1, }, { - description: "should prepare resource. claim in cache, manager did not prepare resource", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ + description: "resource already prepared", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), + claimInfo: genTestClaimInfo([]string{podUID}, true), + expectedClaimInfoState: genClaimInfoState(cdiID), + resp: &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{ + string(claimUID): { + Devices: []*drapb.Device{ { - Name: "test-pod-claim", - ResourceClaimName: func() *string { - s := "test-pod-claim" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim", - }, - }, - }, + PoolName: poolName, + DeviceName: deviceName, + RequestNames: []string{requestName}, + CDIDeviceIDs: []string{cdiID}, }, }, }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClassName: "test-class", - ClaimName: "test-pod-claim", - ClaimUID: "test-reserved", - Namespace: "test-namespace", - PodUIDs: sets.Set[string]{"test-reserved": sets.Empty{}}, - ResourceHandles: []resourceapi.ResourceHandle{{Data: "test-data", DriverName: driverName}}, - }, - annotations: make(map[string][]kubecontainer.Annotation), - prepared: false, - }, - resourceClaim: &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-claim", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: "test-reserved"}, - }, - }, - }, - resp: &drapb.NodePrepareResourcesResponse{ - Claims: map[string]*drapb.NodePrepareResourceResponse{ - "test-reserved": {CDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}}, - }, - }, - expectedCDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}, - ExpectedPrepareCalls: 1, + }}, }, } { t.Run(test.description, func(t *testing.T) { @@ -904,10 +548,13 @@ func TestPrepareResources(t *testing.T) { cache: cache, } - if test.resourceClaim != nil { - if _, err := fakeKubeClient.ResourceV1alpha3().ResourceClaims(test.pod.Namespace).Create(context.Background(), test.resourceClaim, metav1.CreateOptions{}); err != nil { - t.Fatalf("failed to create ResourceClaim %s: %+v", test.resourceClaim.Name, err) + if test.claim != nil { + if _, err := fakeKubeClient.ResourceV1alpha3().ResourceClaims(test.pod.Namespace).Create(context.Background(), test.claim, metav1.CreateOptions{}); err != nil { + t.Fatalf("failed to create ResourceClaim %s: %+v", test.claim.Name, err) } + defer func() { + require.NoError(t, fakeKubeClient.ResourceV1alpha3().ResourceClaims(test.pod.Namespace).Delete(context.Background(), test.claim.Name, metav1.DeleteOptions{})) + }() } var pluginClientTimeout *time.Duration @@ -934,17 +581,22 @@ func TestPrepareResources(t *testing.T) { err = manager.PrepareResources(test.pod) - assert.Equal(t, test.ExpectedPrepareCalls, draServerInfo.server.prepareResourceCalls.Load()) + assert.Equal(t, test.expectedPrepareCalls, draServerInfo.server.prepareResourceCalls.Load()) - if test.wantErr { + if test.expectedErrMsg != "" { assert.Error(t, err) - return // PrepareResources returned an error so stopping the subtest here - } else if test.wantResourceSkipped { - assert.NoError(t, err) - return // resource skipped so no need to continue + if err != nil { + assert.Contains(t, err.Error(), test.expectedErrMsg) + } + return // PrepareResources returned an error so stopping the test case here } assert.NoError(t, err) + + if test.wantResourceSkipped { + return // resource skipped so no need to continue + } + // check the cache contains the expected claim info claimName, _, err := resourceclaim.Name(test.pod, &test.pod.Spec.ResourceClaims[0]) if err != nil { @@ -954,320 +606,84 @@ func TestPrepareResources(t *testing.T) { if !ok { t.Fatalf("claimInfo not found in cache for claim %s", *claimName) } - if claimInfo.DriverName != test.resourceClaim.Status.DriverName { - t.Fatalf("driverName mismatch: expected %s, got %s", test.resourceClaim.Status.DriverName, claimInfo.DriverName) - } - if claimInfo.ClassName != test.resourceClaim.Spec.ResourceClassName { - t.Fatalf("resourceClassName mismatch: expected %s, got %s", test.resourceClaim.Spec.ResourceClassName, claimInfo.ClassName) - } if len(claimInfo.PodUIDs) != 1 || !claimInfo.PodUIDs.Has(string(test.pod.UID)) { t.Fatalf("podUIDs mismatch: expected [%s], got %v", test.pod.UID, claimInfo.PodUIDs) } - assert.ElementsMatchf(t, claimInfo.CDIDevices[test.resourceClaim.Status.DriverName], test.expectedCDIDevices, - "cdiDevices mismatch: expected [%v], got %v", test.expectedCDIDevices, claimInfo.CDIDevices[test.resourceClaim.Status.DriverName]) + + assert.Equal(t, test.expectedClaimInfoState, claimInfo.ClaimInfoState) }) } } func TestUnprepareResources(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset() - for _, test := range []struct { - description string - driverName string - pod *v1.Pod - claimInfo *ClaimInfo - resp *drapb.NodeUnprepareResourcesResponse - wantErr bool - wantTimeout bool - wantResourceSkipped bool + description string + driverName string + pod *v1.Pod + claimInfo *ClaimInfo + claim *resourceapi.ResourceClaim + resp *drapb.NodeUnprepareResourcesResponse + wantTimeout bool + wantResourceSkipped bool + expectedUnprepareCalls uint32 + expectedErrMsg string }{ { - description: "plugin does not exist", - driverName: "this-plugin-does-not-exist", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "another-claim-test", - ResourceClaimName: func() *string { - s := "another-claim-test" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "another-claim-test", - }, - }, - }, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClaimName: "another-claim-test", - Namespace: "test-namespace", - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: driverName, - Data: "test data", - }, - }, - }, - }, - wantErr: true, + description: "unknown driver", + pod: genTestPod(), + claim: genTestClaim(claimName, "unknown driver", deviceName, podUID), + claimInfo: genTestClaimInfo([]string{podUID}, true), + expectedErrMsg: "plugin name test-driver not found in the list of registered DRA plugins", }, { - description: "resource claim referenced by other pod(s)", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-1", - ResourceClaimName: func() *string { - s := "test-pod-claim-1" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-1", - }, - }, - }, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClaimName: "test-pod-claim-1", - Namespace: "test-namespace", - PodUIDs: sets.Set[string]{"test-reserved": sets.Empty{}, "test-reserved-2": sets.Empty{}}, - }, - }, + description: "resource claim referenced by other pod(s)", + driverName: driverName, + pod: genTestPod(), + claimInfo: genTestClaimInfo([]string{podUID, "another-pod-uid"}, true), wantResourceSkipped: true, }, { - description: "should timeout", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-2", - ResourceClaimName: func() *string { - s := "test-pod-claim-2" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-2", - }, - }, - }, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClaimName: "test-pod-claim-2", - Namespace: "test-namespace", - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: driverName, - Data: "test data", - }, - }, - }, - }, - resp: &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"test-reserved": {}}}, - wantErr: true, + description: "should timeout", + driverName: driverName, + pod: genTestPod(), + claimInfo: genTestClaimInfo([]string{podUID}, true), wantTimeout: true, expectedUnprepareCalls: 1, + expectedErrMsg: "context deadline exceeded", }, { - description: "should unprepare resource, claim previously prepared by currently running manager", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-3", - ResourceClaimName: func() *string { - s := "test-pod-claim-3" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-3", - }, - }, - }, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClaimName: "test-pod-claim-3", - Namespace: "test-namespace", - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: driverName, - Data: "test data", - }, - }, - }, - prepared: true, - }, - resp: &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"": {}}}, + description: "should fail when driver returns empty response", + driverName: driverName, + pod: genTestPod(), + claimInfo: genTestClaimInfo([]string{podUID}, true), + resp: &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{}}, + expectedUnprepareCalls: 1, + expectedErrMsg: "NodeUnprepareResources left out 1 claims", + }, + { + description: "should unprepare resource", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), + claimInfo: genTestClaimInfo([]string{podUID}, false), expectedUnprepareCalls: 1, }, { - description: "should unprepare resource, claim previously was not prepared by currently running manager", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim", - ResourceClaimName: func() *string { - s := "test-pod-claim" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim", - }, - }, - }, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClaimName: "test-pod-claim", - Namespace: "test-namespace", - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: driverName, - Data: "test data", - }, - }, - }, - prepared: false, - }, - resp: &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"": {}}}, + description: "should unprepare already prepared resource", + driverName: driverName, + pod: genTestPod(), + claim: genTestClaim(claimName, driverName, deviceName, podUID), + claimInfo: genTestClaimInfo([]string{podUID}, true), expectedUnprepareCalls: 1, }, { - description: "should unprepare resource, driver returns nil value", - driverName: driverName, - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "test-namespace", - UID: "test-reserved", - }, - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "test-pod-claim-nil", - ResourceClaimName: func() *string { - s := "test-pod-claim-nil" - return &s - }(), - }, - }, - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "test-pod-claim-nil", - }, - }, - }, - }, - }, - }, - }, - claimInfo: &ClaimInfo{ - ClaimInfoState: state.ClaimInfoState{ - DriverName: driverName, - ClaimName: "test-pod-claim-nil", - Namespace: "test-namespace", - ClaimUID: "test-reserved", - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: driverName, - Data: "test data", - }, - }, - }, - prepared: true, - }, - resp: &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"test-reserved": nil}}, + description: "should unprepare resource when driver returns nil value", + driverName: driverName, + pod: genTestPod(), + claimInfo: genTestClaimInfo([]string{podUID}, true), + resp: &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{string(claimUID): nil}}, expectedUnprepareCalls: 1, }, } { @@ -1308,15 +724,20 @@ func TestUnprepareResources(t *testing.T) { assert.Equal(t, test.expectedUnprepareCalls, draServerInfo.server.unprepareResourceCalls.Load()) - if test.wantErr { + if test.expectedErrMsg != "" { assert.Error(t, err) - return // UnprepareResources returned an error so stopping the subtest here - } else if test.wantResourceSkipped { - assert.NoError(t, err) - return // resource skipped so no need to continue + if err != nil { + assert.Contains(t, err.Error(), test.expectedErrMsg) + } + return // PrepareResources returned an error so stopping the test case here } assert.NoError(t, err) + + if test.wantResourceSkipped { + return // resource skipped so no need to continue + } + // Check that the cache has been updated correctly claimName, _, err := resourceclaim.Name(test.pod, &test.pod.Spec.ResourceClaims[0]) if err != nil { @@ -1342,10 +763,6 @@ func TestPodMightNeedToUnprepareResources(t *testing.T) { cache: cache, } - claimName := "test-claim" - podUID := "test-pod-uid" - namespace := "test-namespace" - claimInfo := &ClaimInfo{ ClaimInfoState: state.ClaimInfoState{PodUIDs: sets.New(podUID), ClaimName: claimName, Namespace: namespace}, } @@ -1359,81 +776,89 @@ func TestPodMightNeedToUnprepareResources(t *testing.T) { } func TestGetContainerClaimInfos(t *testing.T) { - cache, err := newClaimInfoCache(t.TempDir(), draManagerStateFileName) - if err != nil { - t.Fatalf("error occur:%v", err) - } - manager := &ManagerImpl{ - cache: cache, - } + for _, test := range []struct { + description string + pod *v1.Pod + claimInfo *ClaimInfo - resourceClaimName := "test-resource-claim-1" - resourceClaimName2 := "test-resource-claim-2" - - for i, test := range []struct { expectedClaimName string - pod *v1.Pod - container *v1.Container - claimInfo *ClaimInfo + expectedErrMsg string }{ { - expectedClaimName: resourceClaimName, - pod: &v1.Pod{ - Spec: v1.PodSpec{ - ResourceClaims: []v1.PodResourceClaim{ - { - Name: "claim1", - ResourceClaimName: &resourceClaimName, - }, - }, - }, - }, - container: &v1.Container{ - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ - { - Name: "claim1", - }, - }, - }, - }, - claimInfo: &ClaimInfo{ClaimInfoState: state.ClaimInfoState{ClaimName: resourceClaimName}}, + description: "should get claim info", + expectedClaimName: claimName, + pod: genTestPod(), + claimInfo: genTestClaimInfo([]string{podUID}, false), }, { - expectedClaimName: resourceClaimName2, + description: "should fail when claim info not found", + pod: genTestPod(), + claimInfo: &ClaimInfo{}, + expectedErrMsg: "unable to get claim info for claim ", + }, + { + description: "should fail when none of the supported fields are set", pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + UID: podUID, + }, Spec: v1.PodSpec{ ResourceClaims: []v1.PodResourceClaim{ { - Name: "claim2", - ResourceClaimName: &resourceClaimName2, + Name: claimName, + // missing ResourceClaimName or ResourceClaimTemplateName }, }, - }, - }, - container: &v1.Container{ - Resources: v1.ResourceRequirements{ - Claims: []v1.ResourceClaim{ + Containers: []v1.Container{ { - Name: "claim2", + Resources: v1.ResourceRequirements{ + Claims: []v1.ResourceClaim{ + { + Name: claimName, + }, + }, + }, }, }, }, }, - claimInfo: &ClaimInfo{ClaimInfoState: state.ClaimInfoState{ClaimName: resourceClaimName2}}, + claimInfo: genTestClaimInfo([]string{podUID}, false), + expectedErrMsg: "none of the supported fields are set", + }, + { + description: "should fail when claim info is not cached", + pod: genTestPod(), + expectedErrMsg: "unable to get claim info for claim ", }, } { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - manager.cache.add(test.claimInfo) + t.Run(test.description, func(t *testing.T) { + cache, err := newClaimInfoCache(t.TempDir(), draManagerStateFileName) + if err != nil { + t.Fatalf("error occur:%v", err) + } + manager := &ManagerImpl{ + cache: cache, + } - fakeClaimInfos, err := manager.GetContainerClaimInfos(test.pod, test.container) - assert.NoError(t, err) - assert.Len(t, fakeClaimInfos, 1) - assert.Equal(t, test.expectedClaimName, fakeClaimInfos[0].ClaimInfoState.ClaimName) + if test.claimInfo != nil { + manager.cache.add(test.claimInfo) + } + + claimInfos, err := manager.GetContainerClaimInfos(test.pod, &test.pod.Spec.Containers[0]) + + if test.expectedErrMsg != "" { + assert.Error(t, err) + if err != nil { + assert.Contains(t, err.Error(), test.expectedErrMsg) + } + return + } - manager.cache.delete(test.pod.Spec.ResourceClaims[0].Name, "default") - _, err = manager.GetContainerClaimInfos(test.pod, test.container) assert.NoError(t, err) + assert.Len(t, claimInfos, 1) + assert.Equal(t, test.expectedClaimName, claimInfos[0].ClaimInfoState.ClaimName) }) } } @@ -1476,13 +901,12 @@ func TestParallelPrepareUnprepareResources(t *testing.T) { wgStart.Wait() // Wait to start all goroutines at the same time var err error - nameSpace := "test-namespace-parallel" claimName := fmt.Sprintf("test-pod-claim-%d", goRoutineNum) - podUID := types.UID(fmt.Sprintf("test-reserved-%d", goRoutineNum)) + podUID := types.UID(fmt.Sprintf("test-pod-uid-%d", goRoutineNum)) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("test-pod-%d", goRoutineNum), - Namespace: nameSpace, + Namespace: namespace, UID: podUID, }, Spec: v1.PodSpec{ @@ -1508,30 +932,10 @@ func TestParallelPrepareUnprepareResources(t *testing.T) { }, }, } - resourceClaim := &resourceapi.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: claimName, - Namespace: nameSpace, - UID: types.UID(fmt.Sprintf("claim-%d", goRoutineNum)), - }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: "test-class", - }, - Status: resourceapi.ResourceClaimStatus{ - DriverName: driverName, - Allocation: &resourceapi.AllocationResult{ - ResourceHandles: []resourceapi.ResourceHandle{ - {Data: "test-data", DriverName: driverName}, - }, - }, - ReservedFor: []resourceapi.ResourceClaimConsumerReference{ - {UID: podUID}, - }, - }, - } + claim := genTestClaim(claimName, driverName, deviceName, string(podUID)) - if _, err = fakeKubeClient.ResourceV1alpha3().ResourceClaims(pod.Namespace).Create(context.Background(), resourceClaim, metav1.CreateOptions{}); err != nil { - t.Errorf("failed to create ResourceClaim %s: %+v", resourceClaim.Name, err) + if _, err = fakeKubeClient.ResourceV1alpha3().ResourceClaims(pod.Namespace).Create(context.Background(), claim, metav1.CreateOptions{}); err != nil { + t.Errorf("failed to create ResourceClaim %s: %+v", claim.Name, err) return } diff --git a/pkg/kubelet/cm/dra/plugin/client_test.go b/pkg/kubelet/cm/dra/plugin/client_test.go index ef374bd10c9..a38b9835dc5 100644 --- a/pkg/kubelet/cm/dra/plugin/client_test.go +++ b/pkg/kubelet/cm/dra/plugin/client_test.go @@ -42,7 +42,14 @@ type fakeV1alpha4GRPCServer struct { var _ drapb.NodeServer = &fakeV1alpha4GRPCServer{} func (f *fakeV1alpha4GRPCServer) NodePrepareResources(ctx context.Context, in *drapb.NodePrepareResourcesRequest) (*drapb.NodePrepareResourcesResponse, error) { - return &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"dummy": {CDIDevices: []string{"dummy"}}}}, nil + return &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"claim-uid": { + Devices: []*drapb.Device{ + { + RequestNames: []string{"test-request"}, + CDIDeviceIDs: []string{"test-cdi-id"}, + }, + }, + }}}, nil } func (f *fakeV1alpha4GRPCServer) NodeUnprepareResources(ctx context.Context, in *drapb.NodeUnprepareResourcesRequest) (*drapb.NodeUnprepareResourcesResponse, error) { @@ -136,7 +143,7 @@ func TestGRPCConnIsReused(t *testing.T) { Claims: []*drapb.Claim{ { Namespace: "dummy-namespace", - Uid: "dummy-uid", + UID: "dummy-uid", Name: "dummy-claim", }, }, diff --git a/pkg/kubelet/cm/dra/plugin/plugin.go b/pkg/kubelet/cm/dra/plugin/plugin.go index cab052089d3..9ccd8142f57 100644 --- a/pkg/kubelet/cm/dra/plugin/plugin.go +++ b/pkg/kubelet/cm/dra/plugin/plugin.go @@ -24,6 +24,7 @@ import ( "time" v1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -72,7 +73,7 @@ func NewRegistrationHandler(kubeClient kubernetes.Interface, getNode func() (*v1 } // wipeResourceSlices deletes ResourceSlices of the node, optionally just for a specific driver. -func (h *RegistrationHandler) wipeResourceSlices(pluginName string) { +func (h *RegistrationHandler) wipeResourceSlices(driver string) { if h.kubeClient == nil { return } @@ -97,9 +98,9 @@ func (h *RegistrationHandler) wipeResourceSlices(pluginName string) { logger.Error(err, "Unexpected error checking for node") return false, nil } - fieldSelector := fields.Set{"nodeName": node.Name} - if pluginName != "" { - fieldSelector["driverName"] = pluginName + fieldSelector := fields.Set{resourceapi.ResourceSliceSelectorNodeName: node.Name} + if driver != "" { + fieldSelector[resourceapi.ResourceSliceSelectorDriver] = driver } err = h.kubeClient.ResourceV1alpha3().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: fieldSelector.String()}) diff --git a/pkg/kubelet/cm/dra/state/checkpoint.go b/pkg/kubelet/cm/dra/state/checkpoint.go index 6ea49c62e99..7cce6118182 100644 --- a/pkg/kubelet/cm/dra/state/checkpoint.go +++ b/pkg/kubelet/cm/dra/state/checkpoint.go @@ -18,15 +18,9 @@ package state import ( "encoding/json" - "errors" - "fmt" - "hash/fnv" - "strings" - "k8s.io/apimachinery/pkg/util/dump" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/checksum" - cmerrors "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" ) var _ checkpointmanager.Checkpoint = &DRAManagerCheckpoint{} @@ -40,20 +34,9 @@ type DRAManagerCheckpoint struct { Checksum checksum.Checksum `json:"checksum"` } -// DraManagerCheckpoint struct is an old implementation of the DraManagerCheckpoint -type DRAManagerCheckpointWithoutResourceHandles struct { - Version string `json:"version"` - Entries ClaimInfoStateListWithoutResourceHandles `json:"entries,omitempty"` - Checksum checksum.Checksum `json:"checksum"` -} - // List of claim info to store in checkpoint type ClaimInfoStateList []ClaimInfoState -// List of claim info to store in checkpoint -// TODO: remove in Beta -type ClaimInfoStateListWithoutResourceHandles []ClaimInfoStateWithoutResourceHandles - // NewDRAManagerCheckpoint returns an instance of Checkpoint func NewDRAManagerCheckpoint() *DRAManagerCheckpoint { return &DRAManagerCheckpoint{ @@ -80,48 +63,6 @@ func (dc *DRAManagerCheckpoint) VerifyChecksum() error { ck := dc.Checksum dc.Checksum = 0 err := ck.Verify(dc) - if errors.Is(err, cmerrors.CorruptCheckpointError{}) { - // Verify with old structs without ResourceHandles field - // TODO: remove in Beta - err = verifyChecksumWithoutResourceHandles(dc, ck) - } dc.Checksum = ck return err } - -// verifyChecksumWithoutResourceHandles is a helper function that verifies checksum of the -// checkpoint in the old format, without ResourceHandles field. -// TODO: remove in Beta. -func verifyChecksumWithoutResourceHandles(dc *DRAManagerCheckpoint, checkSum checksum.Checksum) error { - entries := ClaimInfoStateListWithoutResourceHandles{} - for _, entry := range dc.Entries { - entries = append(entries, ClaimInfoStateWithoutResourceHandles{ - DriverName: entry.DriverName, - ClassName: entry.ClassName, - ClaimUID: entry.ClaimUID, - ClaimName: entry.ClaimName, - Namespace: entry.Namespace, - PodUIDs: entry.PodUIDs, - CDIDevices: entry.CDIDevices, - }) - } - oldcheckpoint := &DRAManagerCheckpointWithoutResourceHandles{ - Version: checkpointVersion, - Entries: entries, - Checksum: 0, - } - // Calculate checksum for old checkpoint - object := dump.ForHash(oldcheckpoint) - object = strings.Replace(object, "DRAManagerCheckpointWithoutResourceHandles", "DRAManagerCheckpoint", 1) - object = strings.Replace(object, "ClaimInfoStateListWithoutResourceHandles", "ClaimInfoStateList", 1) - hash := fnv.New32a() - fmt.Fprintf(hash, "%v", object) - actualCS := checksum.Checksum(hash.Sum32()) - if checkSum != actualCS { - return &cmerrors.CorruptCheckpointError{ - ActualCS: uint64(actualCS), - ExpectedCS: uint64(checkSum), - } - } - return nil -} diff --git a/pkg/kubelet/cm/dra/state/state_checkpoint.go b/pkg/kubelet/cm/dra/state/state_checkpoint.go index 0da7e8c78b5..ce20e1ffdf9 100644 --- a/pkg/kubelet/cm/dra/state/state_checkpoint.go +++ b/pkg/kubelet/cm/dra/state/state_checkpoint.go @@ -37,12 +37,6 @@ type CheckpointState interface { // ClaimInfoState is used to store claim info state in a checkpoint // +k8s:deepcopy-gen=true type ClaimInfoState struct { - // Name of the DRA driver - DriverName string - - // ClassName is a resource class of the claim - ClassName string - // ClaimUID is an UID of the resource claim ClaimUID types.UID @@ -55,35 +49,25 @@ type ClaimInfoState struct { // PodUIDs is a set of pod UIDs that reference a resource PodUIDs sets.Set[string] - // CDIDevices is a map of DriverName --> CDI devices returned by the - // GRPC API call NodePrepareResource - CDIDevices map[string][]string + // DriverState contains information about all drivers which have allocation + // results in the claim, even if they don't provide devices for their results. + DriverState map[string]DriverState } -// ClaimInfoStateWithoutResourceHandles is an old implementation of the ClaimInfoState -// TODO: remove in Beta -type ClaimInfoStateWithoutResourceHandles struct { - // Name of the DRA driver - DriverName string +// DriverState is used to store per-device claim info state in a checkpoint +// +k8s:deepcopy-gen=true +type DriverState struct { + Devices []Device +} - // ClassName is a resource class of the claim - ClassName string - - // ClaimUID is an UID of the resource claim - ClaimUID types.UID - - // ClaimName is a name of the resource claim - ClaimName string - - // Namespace is a claim namespace - Namespace string - - // PodUIDs is a set of pod UIDs that reference a resource - PodUIDs sets.Set[string] - - // CDIDevices is a map of DriverName --> CDI devices returned by the - // GRPC API call NodePrepareResource - CDIDevices map[string][]string +// Device is how a DRA driver described an allocated device in a claim +// to kubelet. RequestName and CDI device IDs are optional. +// +k8s:deepcopy-gen=true +type Device struct { + PoolName string + DeviceName string + RequestNames []string + CDIDeviceIDs []string } type stateCheckpoint struct { diff --git a/pkg/kubelet/cm/dra/state/state_checkpoint_test.go b/pkg/kubelet/cm/dra/state/state_checkpoint_test.go index 2fa1dc5ee8f..4668235d40b 100644 --- a/pkg/kubelet/cm/dra/state/state_checkpoint_test.go +++ b/pkg/kubelet/cm/dra/state/state_checkpoint_test.go @@ -19,14 +19,12 @@ package state import ( "errors" "os" - "path" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - resourceapi "k8s.io/api/resource/v1alpha3" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" cmerrors "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" @@ -55,116 +53,120 @@ func TestCheckpointGetOrCreate(t *testing.T) { expectedState ClaimInfoStateList }{ { - "Create non-existing checkpoint", - "", - "", - []ClaimInfoState{}, + description: "Create non-existing checkpoint", + checkpointContent: "", + expectedError: "", + expectedState: []ClaimInfoState{}, }, { - "Restore checkpoint - single claim", - `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"ResourceHandles":[{"driverName":"test-driver.cdi.k8s.io","data":"{\"a\": \"b\"}"}],"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":113577689}`, - "", - []ClaimInfoState{ + description: "Restore checkpoint - single claim", + checkpointContent: `{"version":"v1","entries":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}}],"checksum":1925941526}`, + expectedState: []ClaimInfoState{ { - DriverName: "test-driver.cdi.k8s.io", - ClassName: "class-name", - ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", - ClaimName: "example", - Namespace: "default", - PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: "test-driver.cdi.k8s.io", - Data: `{"a": "b"}`, + DriverState: map[string]DriverState{ + "test-driver.cdi.k8s.io": { + Devices: []Device{ + { + PoolName: "worker-1", + DeviceName: "dev-1", + RequestNames: []string{"test request"}, + CDIDeviceIDs: []string{"example.com/example=cdi-example"}, + }, + }, }, }, - CDIDevices: map[string][]string{ - "test-driver.cdi.k8s.io": {"example.com/example=cdi-example"}, - }, + ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", + ClaimName: "example", + Namespace: "default", + PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, }, { - "Restore checkpoint - single claim - multiple devices", - `{"version":"v1","entries":[{"DriverName":"meta-test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"ResourceHandles":[{"driverName":"test-driver-1.cdi.k8s.io","data":"{\"a\": \"b\"}"},{"driverName":"test-driver-2.cdi.k8s.io","data":"{\"c\": \"d\"}"}],"CDIDevices":{"test-driver-1.cdi.k8s.io":["example-1.com/example-1=cdi-example-1"],"test-driver-2.cdi.k8s.io":["example-2.com/example-2=cdi-example-2"]}}],"checksum":1466990255}`, - "", - []ClaimInfoState{ + description: "Restore checkpoint - single claim - multiple devices", + checkpointContent: `{"version":"v1","entries":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]},{"PoolName":"worker-1","DeviceName":"dev-2","RequestNames":["test request2"],"CDIDeviceIDs":["example.com/example=cdi-example2"]}]}}}],"checksum":3560752542}`, + expectedError: "", + expectedState: []ClaimInfoState{ { - DriverName: "meta-test-driver.cdi.k8s.io", - ClassName: "class-name", - ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", - ClaimName: "example", - Namespace: "default", - PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: "test-driver-1.cdi.k8s.io", - Data: `{"a": "b"}`, - }, - { - DriverName: "test-driver-2.cdi.k8s.io", - Data: `{"c": "d"}`, + DriverState: map[string]DriverState{ + "test-driver.cdi.k8s.io": { + Devices: []Device{ + { + PoolName: "worker-1", + DeviceName: "dev-1", + RequestNames: []string{"test request"}, + CDIDeviceIDs: []string{"example.com/example=cdi-example"}, + }, + { + PoolName: "worker-1", + DeviceName: "dev-2", + RequestNames: []string{"test request2"}, + CDIDeviceIDs: []string{"example.com/example=cdi-example2"}, + }, + }, }, }, - CDIDevices: map[string][]string{ - "test-driver-1.cdi.k8s.io": {"example-1.com/example-1=cdi-example-1"}, - "test-driver-2.cdi.k8s.io": {"example-2.com/example-2=cdi-example-2"}, - }, + ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", + ClaimName: "example", + Namespace: "default", + PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, }, { - "Restore checkpoint - multiple claims", - `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name-1","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"ResourceHandles":[{"driverName":"test-driver.cdi.k8s.io","data":"{\"a\": \"b\"}"}],"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example-1"]}},{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name-2","ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"ResourceHandles":[{"driverName":"test-driver.cdi.k8s.io","data":"{\"c\": \"d\"}"}],"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example-2"]}}],"checksum":471181742}`, - "", - []ClaimInfoState{ + description: "Restore checkpoint - multiple claims", + checkpointContent: `{"version":"v1","entries":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}},{"ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}}],"checksum":351581974}`, + expectedError: "", + expectedState: []ClaimInfoState{ { - DriverName: "test-driver.cdi.k8s.io", - ClassName: "class-name-1", - ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", - ClaimName: "example-1", - Namespace: "default", - PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: "test-driver.cdi.k8s.io", - Data: `{"a": "b"}`, + DriverState: map[string]DriverState{ + "test-driver.cdi.k8s.io": { + Devices: []Device{ + { + PoolName: "worker-1", + DeviceName: "dev-1", + RequestNames: []string{"test request"}, + CDIDeviceIDs: []string{"example.com/example=cdi-example"}, + }, + }, }, }, - CDIDevices: map[string][]string{ - "test-driver.cdi.k8s.io": {"example.com/example=cdi-example-1"}, - }, + ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", + ClaimName: "example-1", + Namespace: "default", + PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, { - DriverName: "test-driver.cdi.k8s.io", - ClassName: "class-name-2", - ClaimUID: "4cf8db2d-06c0-7d70-1a51-e59b25b2c16c", - ClaimName: "example-2", - Namespace: "default", - PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: "test-driver.cdi.k8s.io", - Data: `{"c": "d"}`, + DriverState: map[string]DriverState{ + "test-driver.cdi.k8s.io": { + Devices: []Device{ + { + PoolName: "worker-1", + DeviceName: "dev-1", + RequestNames: []string{"test request"}, + CDIDeviceIDs: []string{"example.com/example=cdi-example"}, + }, + }, }, }, - CDIDevices: map[string][]string{ - "test-driver.cdi.k8s.io": {"example.com/example=cdi-example-2"}, - }, + ClaimUID: "4cf8db2d-06c0-7d70-1a51-e59b25b2c16c", + ClaimName: "example-2", + Namespace: "default", + PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, }, { - "Restore checkpoint - invalid checksum", - `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":1988120168}`, - "checkpoint is corrupted", - []ClaimInfoState{}, + description: "Restore checkpoint - invalid checksum", + checkpointContent: `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":1988120168}`, + expectedError: "checkpoint is corrupted", + expectedState: []ClaimInfoState{}, }, { - "Restore checkpoint with invalid JSON", - `{`, - "unexpected end of JSON input", - []ClaimInfoState{}, + description: "Restore checkpoint with invalid JSON", + checkpointContent: `{`, + expectedError: "unexpected end of JSON input", + expectedState: []ClaimInfoState{}, }, } @@ -198,8 +200,7 @@ func TestCheckpointGetOrCreate(t *testing.T) { state, err = checkpointState.GetOrCreate() } if strings.TrimSpace(tc.expectedError) != "" { - assert.Error(t, err) - assert.Contains(t, err.Error(), tc.expectedError) + assert.ErrorContains(t, err, tc.expectedError) } else { requireNoCheckpointError(t, err) // compare state after restoration with the one expected @@ -210,25 +211,40 @@ func TestCheckpointGetOrCreate(t *testing.T) { } func TestCheckpointStateStore(t *testing.T) { - claimInfoState := ClaimInfoState{ - DriverName: "test-driver.cdi.k8s.io", - ClassName: "class-name", - ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", - ClaimName: "example", - Namespace: "default", - PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), - ResourceHandles: []resourceapi.ResourceHandle{ - { - DriverName: "test-driver.cdi.k8s.io", - Data: `{"a": "b"}`, + claimInfoStateList := ClaimInfoStateList{ + { + DriverState: map[string]DriverState{ + "test-driver.cdi.k8s.io": { + Devices: []Device{{ + PoolName: "worker-1", + DeviceName: "dev-1", + RequestNames: []string{"test request"}, + CDIDeviceIDs: []string{"example.com/example=cdi-example"}, + }}, + }, }, + ClaimUID: "067798be-454e-4be4-9047-1aa06aea63f7", + ClaimName: "example-1", + Namespace: "default", + PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, - CDIDevices: map[string][]string{ - "test-driver.cdi.k8s.io": {"example.com/example=cdi-example"}, + { + DriverState: map[string]DriverState{ + "test-driver.cdi.k8s.io": { + Devices: []Device{{ + PoolName: "worker-1", + DeviceName: "dev-2", + }}, + }, + }, + ClaimUID: "4cf8db2d-06c0-7d70-1a51-e59b25b2c16c", + ClaimName: "example-2", + Namespace: "default", + PodUIDs: sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, } - expectedCheckpoint := `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"ResourceHandles":[{"driverName":"test-driver.cdi.k8s.io","data":"{\"a\": \"b\"}"}],"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":113577689}` + expectedCheckpoint := `{"version":"v1","entries":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}},{"ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-2","RequestNames":null,"CDIDeviceIDs":null}]}}}],"checksum":1191151426}` // Should return an error, stateDir cannot be an empty string if _, err := NewCheckpointState("", testingCheckpoint); err == nil { @@ -248,7 +264,7 @@ func TestCheckpointStateStore(t *testing.T) { cs, err := NewCheckpointState(testingDir, testingCheckpoint) assert.NoError(t, err, "could not create testing checkpointState instance") - err = cs.Store(ClaimInfoStateList{claimInfoState}) + err = cs.Store(claimInfoStateList) assert.NoError(t, err, "could not store ClaimInfoState") checkpoint := NewDRAManagerCheckpoint() cpm.GetCheckpoint(testingCheckpoint, checkpoint) @@ -262,26 +278,6 @@ func TestCheckpointStateStore(t *testing.T) { } } -func TestOldCheckpointRestore(t *testing.T) { - testingDir := t.TempDir() - cpm, err := checkpointmanager.NewCheckpointManager(testingDir) - assert.NoError(t, err, "could not create testing checkpoint manager") - - oldCheckpointData := `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":153446146}` - err = os.WriteFile(path.Join(testingDir, testingCheckpoint), []byte(oldCheckpointData), 0644) - assert.NoError(t, err, "could not store checkpoint data") - - checkpoint := NewDRAManagerCheckpoint() - err = cpm.GetCheckpoint(testingCheckpoint, checkpoint) - requireNoCheckpointError(t, err) - - checkpointData, err := checkpoint.MarshalCheckpoint() - assert.NoError(t, err, "could not Marshal Checkpoint") - - expectedData := `{"version":"v1","entries":[{"DriverName":"test-driver.cdi.k8s.io","ClassName":"class-name","ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"ResourceHandles":null,"CDIDevices":{"test-driver.cdi.k8s.io":["example.com/example=cdi-example"]}}],"checksum":453625682}` - assert.Equal(t, expectedData, string(checkpointData), "expected ClaimInfoState does not equal to restored one") -} - func requireNoCheckpointError(t *testing.T, err error) { t.Helper() var cksumErr *cmerrors.CorruptCheckpointError diff --git a/pkg/kubelet/cm/dra/state/zz_generated.deepcopy.go b/pkg/kubelet/cm/dra/state/zz_generated.deepcopy.go index a0ab5191dc4..6489dab7a51 100644 --- a/pkg/kubelet/cm/dra/state/zz_generated.deepcopy.go +++ b/pkg/kubelet/cm/dra/state/zz_generated.deepcopy.go @@ -35,19 +35,11 @@ func (in *ClaimInfoState) DeepCopyInto(out *ClaimInfoState) { (*out)[key] = val } } - if in.CDIDevices != nil { - in, out := &in.CDIDevices, &out.CDIDevices - *out = make(map[string][]string, len(*in)) + if in.DriverState != nil { + in, out := &in.DriverState, &out.DriverState + *out = make(map[string]DriverState, len(*in)) for key, val := range *in { - var outVal []string - if val == nil { - (*out)[key] = nil - } else { - in, out := &val, &outVal - *out = make([]string, len(*in)) - copy(*out, *in) - } - (*out)[key] = outVal + (*out)[key] = *val.DeepCopy() } } return @@ -62,3 +54,52 @@ func (in *ClaimInfoState) DeepCopy() *ClaimInfoState { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Device) DeepCopyInto(out *Device) { + *out = *in + if in.RequestNames != nil { + in, out := &in.RequestNames, &out.RequestNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CDIDeviceIDs != nil { + in, out := &in.CDIDeviceIDs, &out.CDIDeviceIDs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Device. +func (in *Device) DeepCopy() *Device { + if in == nil { + return nil + } + out := new(Device) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DriverState) DeepCopyInto(out *DriverState) { + *out = *in + if in.Devices != nil { + in, out := &in.Devices, &out.Devices + *out = make([]Device, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriverState. +func (in *DriverState) DeepCopy() *DriverState { + if in == nil { + return nil + } + out := new(DriverState) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/kubelet/cm/dra/types.go b/pkg/kubelet/cm/dra/types.go index e009e952eb4..6b37d5039b5 100644 --- a/pkg/kubelet/cm/dra/types.go +++ b/pkg/kubelet/cm/dra/types.go @@ -50,8 +50,6 @@ type Manager interface { // ContainerInfo contains information required by the runtime to consume prepared resources. type ContainerInfo struct { - // The Annotations for the container - Annotations []kubecontainer.Annotation // CDI Devices for the container CDIDevices []kubecontainer.CDIDevice } diff --git a/pkg/kubelet/cm/dra/zz_generated.deepcopy.go b/pkg/kubelet/cm/dra/zz_generated.deepcopy.go index cc10fdaf53e..577fd431881 100644 --- a/pkg/kubelet/cm/dra/zz_generated.deepcopy.go +++ b/pkg/kubelet/cm/dra/zz_generated.deepcopy.go @@ -21,29 +21,10 @@ limitations under the License. package dra -import ( - container "k8s.io/kubernetes/pkg/kubelet/container" -) - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClaimInfo) DeepCopyInto(out *ClaimInfo) { *out = *in in.ClaimInfoState.DeepCopyInto(&out.ClaimInfoState) - if in.annotations != nil { - in, out := &in.annotations, &out.annotations - *out = make(map[string][]container.Annotation, len(*in)) - for key, val := range *in { - var outVal []container.Annotation - if val == nil { - (*out)[key] = nil - } else { - in, out := &val, &outVal - *out = make([]container.Annotation, len(*in)) - copy(*out, *in) - } - (*out)[key] = outVal - } - } return } diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index b29b89f0778..ecdf8f902e8 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -492,7 +492,8 @@ type DeviceInfo struct { // CDIDevice contains information about CDI device type CDIDevice struct { - // Name is a fully qualified device name + // Name is a fully qualified device name according to + // https://github.com/cncf-tags/container-device-interface/blob/e66544063aa7760c4ea6330ce9e6c757f8e61df2/README.md?plain=1#L9-L15 Name string } diff --git a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.pb.go b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.pb.go index 5c96e3edda5..581c708b732 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.pb.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.pb.go @@ -145,9 +145,10 @@ func (m *NodePrepareResourcesResponse) GetClaims() map[string]*NodePrepareResour type NodePrepareResourceResponse struct { // These are the additional devices that kubelet must - // make available via the container runtime. A resource + // make available via the container runtime. A claim + // may have zero or more requests and each request // may have zero or more devices. - CDIDevices []string `protobuf:"bytes,1,rep,name=cdi_devices,json=cdiDevices,proto3" json:"cdi_devices,omitempty"` + Devices []*Device `protobuf:"bytes,1,rep,name=devices,proto3" json:"devices,omitempty"` // If non-empty, preparing the ResourceClaim failed. // cdi_devices is ignored in that case. Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` @@ -187,9 +188,9 @@ func (m *NodePrepareResourceResponse) XXX_DiscardUnknown() { var xxx_messageInfo_NodePrepareResourceResponse proto.InternalMessageInfo -func (m *NodePrepareResourceResponse) GetCDIDevices() []string { +func (m *NodePrepareResourceResponse) GetDevices() []*Device { if m != nil { - return m.CDIDevices + return m.Devices } return nil } @@ -201,6 +202,81 @@ func (m *NodePrepareResourceResponse) GetError() string { return "" } +type Device struct { + // The requests in the claim that this device is associated with. + // Optional. If empty, the device is associated with all requests. + RequestNames []string `protobuf:"bytes,1,rep,name=request_names,json=requestNames,proto3" json:"request_names,omitempty"` + // The pool which contains the device. Required. + PoolName string `protobuf:"bytes,2,opt,name=pool_name,json=poolName,proto3" json:"pool_name,omitempty"` + // The device itself. Required. + DeviceName string `protobuf:"bytes,3,opt,name=device_name,json=deviceName,proto3" json:"device_name,omitempty"` + // A single device instance may map to several CDI device IDs. + // None is also valid. + CDIDeviceIDs []string `protobuf:"bytes,4,rep,name=cdi_device_ids,json=cdiDeviceIds,proto3" json:"cdi_device_ids,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Device) Reset() { *m = Device{} } +func (*Device) ProtoMessage() {} +func (*Device) Descriptor() ([]byte, []int) { + return fileDescriptor_00212fb1f9d3bf1c, []int{3} +} +func (m *Device) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Device) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Device.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Device) XXX_Merge(src proto.Message) { + xxx_messageInfo_Device.Merge(m, src) +} +func (m *Device) XXX_Size() int { + return m.Size() +} +func (m *Device) XXX_DiscardUnknown() { + xxx_messageInfo_Device.DiscardUnknown(m) +} + +var xxx_messageInfo_Device proto.InternalMessageInfo + +func (m *Device) GetRequestNames() []string { + if m != nil { + return m.RequestNames + } + return nil +} + +func (m *Device) GetPoolName() string { + if m != nil { + return m.PoolName + } + return "" +} + +func (m *Device) GetDeviceName() string { + if m != nil { + return m.DeviceName + } + return "" +} + +func (m *Device) GetCDIDeviceIDs() []string { + if m != nil { + return m.CDIDeviceIDs + } + return nil +} + type NodeUnprepareResourcesRequest struct { // The list of ResourceClaims that are to be unprepared. Claims []*Claim `protobuf:"bytes,1,rep,name=claims,proto3" json:"claims,omitempty"` @@ -211,7 +287,7 @@ type NodeUnprepareResourcesRequest struct { func (m *NodeUnprepareResourcesRequest) Reset() { *m = NodeUnprepareResourcesRequest{} } func (*NodeUnprepareResourcesRequest) ProtoMessage() {} func (*NodeUnprepareResourcesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{3} + return fileDescriptor_00212fb1f9d3bf1c, []int{4} } func (m *NodeUnprepareResourcesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -259,7 +335,7 @@ type NodeUnprepareResourcesResponse struct { func (m *NodeUnprepareResourcesResponse) Reset() { *m = NodeUnprepareResourcesResponse{} } func (*NodeUnprepareResourcesResponse) ProtoMessage() {} func (*NodeUnprepareResourcesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{4} + return fileDescriptor_00212fb1f9d3bf1c, []int{5} } func (m *NodeUnprepareResourcesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -305,7 +381,7 @@ type NodeUnprepareResourceResponse struct { func (m *NodeUnprepareResourceResponse) Reset() { *m = NodeUnprepareResourceResponse{} } func (*NodeUnprepareResourceResponse) ProtoMessage() {} func (*NodeUnprepareResourceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{5} + return fileDescriptor_00212fb1f9d3bf1c, []int{6} } func (m *NodeUnprepareResourceResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -347,7 +423,7 @@ type Claim struct { Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` // The UID of the Resource claim (ResourceClaim.meta.UUID). // This field is REQUIRED. - Uid string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"` + UID string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"` // The name of the Resource claim (ResourceClaim.meta.Name) // This field is REQUIRED. Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` @@ -358,7 +434,7 @@ type Claim struct { func (m *Claim) Reset() { *m = Claim{} } func (*Claim) ProtoMessage() {} func (*Claim) Descriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{6} + return fileDescriptor_00212fb1f9d3bf1c, []int{7} } func (m *Claim) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -394,9 +470,9 @@ func (m *Claim) GetNamespace() string { return "" } -func (m *Claim) GetUid() string { +func (m *Claim) GetUID() string { if m != nil { - return m.Uid + return m.UID } return "" } @@ -413,6 +489,7 @@ func init() { proto.RegisterType((*NodePrepareResourcesResponse)(nil), "v1alpha3.NodePrepareResourcesResponse") proto.RegisterMapType((map[string]*NodePrepareResourceResponse)(nil), "v1alpha3.NodePrepareResourcesResponse.ClaimsEntry") proto.RegisterType((*NodePrepareResourceResponse)(nil), "v1alpha3.NodePrepareResourceResponse") + proto.RegisterType((*Device)(nil), "v1alpha3.Device") proto.RegisterType((*NodeUnprepareResourcesRequest)(nil), "v1alpha3.NodeUnprepareResourcesRequest") proto.RegisterType((*NodeUnprepareResourcesResponse)(nil), "v1alpha3.NodeUnprepareResourcesResponse") proto.RegisterMapType((map[string]*NodeUnprepareResourceResponse)(nil), "v1alpha3.NodeUnprepareResourcesResponse.ClaimsEntry") @@ -423,38 +500,43 @@ func init() { func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } var fileDescriptor_00212fb1f9d3bf1c = []byte{ - // 481 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0xcd, 0x36, 0x4d, 0x85, 0x27, 0x12, 0xa0, 0x55, 0x85, 0xa2, 0x50, 0x4c, 0x64, 0x51, 0x92, - 0x0b, 0xb6, 0x48, 0x8b, 0x54, 0x81, 0xb8, 0xa4, 0x05, 0xf1, 0x25, 0x84, 0x2c, 0x71, 0xe1, 0x02, - 0x6b, 0x7b, 0x70, 0x57, 0xf9, 0xd8, 0x65, 0xd7, 0x8e, 0xd4, 0x1b, 0x3f, 0x81, 0x9f, 0xd5, 0x03, - 0x07, 0xc4, 0x89, 0x53, 0x45, 0xcd, 0x1f, 0x41, 0x5e, 0x3b, 0xe9, 0x87, 0x9c, 0x26, 0x52, 0x6f, - 0x33, 0xe3, 0x9d, 0x79, 0x6f, 0xde, 0x1b, 0x19, 0x2c, 0x26, 0xb9, 0x2b, 0x95, 0x48, 0x04, 0xbd, - 0x31, 0x7d, 0xcc, 0x46, 0xf2, 0x90, 0xed, 0xb4, 0x1f, 0xc5, 0x3c, 0x39, 0x4c, 0x03, 0x37, 0x14, - 0x63, 0x2f, 0x16, 0xb1, 0xf0, 0xcc, 0x83, 0x20, 0xfd, 0x6a, 0x32, 0x93, 0x98, 0xa8, 0x68, 0x74, - 0x5e, 0xc2, 0xdd, 0xf7, 0x22, 0xc2, 0x0f, 0x0a, 0x25, 0x53, 0xe8, 0xa3, 0x16, 0xa9, 0x0a, 0x51, - 0xfb, 0xf8, 0x2d, 0x45, 0x9d, 0xd0, 0x2e, 0x6c, 0x84, 0x23, 0xc6, 0xc7, 0xba, 0x45, 0x3a, 0xf5, - 0x5e, 0xb3, 0x7f, 0xcb, 0x9d, 0x01, 0xb9, 0xfb, 0x79, 0xdd, 0x2f, 0x3f, 0x3b, 0x3f, 0x09, 0x6c, - 0x55, 0x0f, 0xd2, 0x52, 0x4c, 0x34, 0xd2, 0x37, 0x97, 0x26, 0xf5, 0xcf, 0x26, 0x5d, 0xd5, 0x57, - 0xc0, 0xe8, 0x17, 0x93, 0x44, 0x1d, 0xcd, 0xc0, 0xda, 0x5f, 0xa0, 0x79, 0xae, 0x4c, 0x6f, 0x43, - 0x7d, 0x88, 0x47, 0x2d, 0xd2, 0x21, 0x3d, 0xcb, 0xcf, 0x43, 0xfa, 0x0c, 0x1a, 0x53, 0x36, 0x4a, - 0xb1, 0xb5, 0xd6, 0x21, 0xbd, 0x66, 0x7f, 0xfb, 0x4a, 0xac, 0x19, 0x94, 0x5f, 0xf4, 0x3c, 0x5d, - 0xdb, 0x23, 0x4e, 0x54, 0x29, 0xcb, 0x7c, 0x19, 0x0f, 0x9a, 0x61, 0xc4, 0x3f, 0x47, 0x38, 0xe5, - 0x21, 0x16, 0x1b, 0x59, 0x83, 0x9b, 0xd9, 0xc9, 0x7d, 0xd8, 0x3f, 0x78, 0x7d, 0x50, 0x54, 0x7d, - 0x08, 0x23, 0x5e, 0xc6, 0x74, 0x13, 0x1a, 0xa8, 0x94, 0x50, 0x86, 0x90, 0xe5, 0x17, 0x89, 0xf3, - 0x0a, 0xee, 0xe5, 0x28, 0x1f, 0x27, 0xf2, 0xba, 0xf2, 0xff, 0x26, 0x60, 0x2f, 0x1a, 0x55, 0x72, - 0x7e, 0x77, 0x69, 0xd6, 0xee, 0x45, 0x51, 0x16, 0x77, 0x56, 0x5a, 0x10, 0x2c, 0xb3, 0xe0, 0xf9, - 0x45, 0x0b, 0xba, 0x4b, 0xd0, 0xaa, 0x4c, 0x78, 0xb2, 0x40, 0x9e, 0xf9, 0x4a, 0x73, 0x55, 0xc9, - 0x79, 0x55, 0xdf, 0x42, 0xc3, 0x50, 0xa3, 0x5b, 0x60, 0x4d, 0xd8, 0x18, 0xb5, 0x64, 0x21, 0x96, - 0x4f, 0xce, 0x0a, 0x39, 0xe5, 0x94, 0x47, 0xa5, 0x21, 0x79, 0x48, 0x29, 0xac, 0xe7, 0x9f, 0x5b, - 0x75, 0x53, 0x32, 0x71, 0xff, 0x84, 0xc0, 0x7a, 0x4e, 0x82, 0xc6, 0xb0, 0x59, 0x75, 0xa7, 0x74, - 0x7b, 0xd9, 0x1d, 0x1b, 0x27, 0xdb, 0x0f, 0x57, 0x3b, 0x77, 0xa7, 0x46, 0xc7, 0x70, 0xa7, 0xda, - 0x0f, 0xda, 0x5d, 0xee, 0x58, 0x01, 0xd6, 0x5b, 0xd5, 0x5a, 0xa7, 0x36, 0x18, 0x1c, 0x9f, 0xda, - 0xe4, 0xcf, 0xa9, 0x5d, 0xfb, 0x9e, 0xd9, 0xe4, 0x38, 0xb3, 0xc9, 0xaf, 0xcc, 0x26, 0x7f, 0x33, - 0x9b, 0xfc, 0xf8, 0x67, 0xd7, 0x3e, 0x3d, 0x18, 0xee, 0x69, 0x97, 0x0b, 0x6f, 0x98, 0x06, 0x38, - 0xc2, 0xc4, 0x93, 0xc3, 0xd8, 0x63, 0x92, 0x6b, 0x2f, 0x52, 0xcc, 0x2b, 0x41, 0x76, 0x83, 0x0d, - 0xf3, 0x2f, 0xd9, 0xf9, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x65, 0xc5, 0xc2, 0x0e, 0x91, 0x04, 0x00, - 0x00, + // 566 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xcb, 0x6e, 0xd3, 0x4c, + 0x14, 0xce, 0x34, 0x97, 0xd6, 0x27, 0xf9, 0x7f, 0xa2, 0x51, 0x85, 0x42, 0x5a, 0x9c, 0xc8, 0x50, + 0x12, 0x21, 0x11, 0x8b, 0xb4, 0xa0, 0x0a, 0xc4, 0x26, 0x0d, 0x88, 0x20, 0x54, 0x21, 0x8b, 0x6e, + 0xd8, 0x04, 0xc7, 0x1e, 0x52, 0x2b, 0x97, 0x19, 0x66, 0xe2, 0x48, 0xdd, 0xf1, 0x08, 0xbc, 0x03, + 0x2f, 0xd3, 0x05, 0x0b, 0xc4, 0x8a, 0x55, 0x44, 0xcd, 0x8b, 0x20, 0xcf, 0xd8, 0x49, 0x5a, 0x39, + 0x4d, 0x25, 0x76, 0x73, 0xbe, 0x73, 0xf9, 0xce, 0x9c, 0xef, 0xcc, 0x80, 0x66, 0x33, 0xaf, 0xc1, + 0x38, 0x9d, 0x50, 0xbc, 0x35, 0x7d, 0x6c, 0x0f, 0xd9, 0xa9, 0xbd, 0x5f, 0x7e, 0xd4, 0xf7, 0x26, + 0xa7, 0x7e, 0xaf, 0xe1, 0xd0, 0x91, 0xd9, 0xa7, 0x7d, 0x6a, 0xca, 0x80, 0x9e, 0xff, 0x49, 0x5a, + 0xd2, 0x90, 0x27, 0x95, 0x68, 0xbc, 0x82, 0x9d, 0x63, 0xea, 0x92, 0x77, 0x9c, 0x30, 0x9b, 0x13, + 0x8b, 0x08, 0xea, 0x73, 0x87, 0x08, 0x8b, 0x7c, 0xf6, 0x89, 0x98, 0xe0, 0x1a, 0xe4, 0x9c, 0xa1, + 0xed, 0x8d, 0x44, 0x09, 0x55, 0xd3, 0xf5, 0x7c, 0xf3, 0x56, 0x23, 0x26, 0x6a, 0x1c, 0x85, 0xb8, + 0x15, 0xb9, 0x8d, 0xef, 0x08, 0x76, 0x93, 0x0b, 0x09, 0x46, 0xc7, 0x82, 0xe0, 0x37, 0x57, 0x2a, + 0x35, 0x17, 0x95, 0xae, 0xcb, 0x53, 0x34, 0xe2, 0xe5, 0x78, 0xc2, 0xcf, 0x62, 0xb2, 0xf2, 0x47, + 0xc8, 0x2f, 0xc1, 0xb8, 0x08, 0xe9, 0x01, 0x39, 0x2b, 0xa1, 0x2a, 0xaa, 0x6b, 0x56, 0x78, 0xc4, + 0xcf, 0x21, 0x3b, 0xb5, 0x87, 0x3e, 0x29, 0x6d, 0x54, 0x51, 0x3d, 0xdf, 0xdc, 0xbb, 0x96, 0x2b, + 0xa6, 0xb2, 0x54, 0xce, 0xb3, 0x8d, 0x43, 0x64, 0x74, 0x13, 0xc7, 0x32, 0xbf, 0xcc, 0x43, 0xd8, + 0x74, 0xc9, 0xd4, 0x73, 0x48, 0x7c, 0x9b, 0xe2, 0x82, 0xa1, 0x2d, 0x1d, 0x56, 0x1c, 0x80, 0xb7, + 0x21, 0x4b, 0x38, 0xa7, 0x5c, 0xf6, 0xa2, 0x59, 0xca, 0x30, 0xbe, 0x21, 0xc8, 0xa9, 0x48, 0x7c, + 0x0f, 0xfe, 0xe3, 0x6a, 0xdc, 0xdd, 0xb1, 0x3d, 0x8a, 0x4a, 0x6a, 0x56, 0x21, 0x02, 0x8f, 0x43, + 0x0c, 0xef, 0x80, 0xc6, 0x28, 0x1d, 0xca, 0x88, 0xa8, 0xd2, 0x56, 0x08, 0x84, 0x5e, 0x5c, 0x81, + 0xbc, 0x62, 0x53, 0xee, 0xb4, 0x74, 0x83, 0x82, 0x64, 0xc0, 0x53, 0xf8, 0xdf, 0x71, 0xbd, 0x6e, + 0x14, 0xe4, 0xb9, 0xa2, 0x94, 0x09, 0x39, 0x5a, 0xc5, 0x60, 0x56, 0x29, 0x1c, 0xb5, 0x3b, 0xaa, + 0x93, 0x4e, 0x5b, 0x58, 0x05, 0xc7, 0xf5, 0x22, 0xcb, 0x15, 0xc6, 0x6b, 0xb8, 0x1b, 0x8e, 0xe1, + 0x64, 0xcc, 0xfe, 0x75, 0x3f, 0x7e, 0x22, 0xd0, 0x57, 0x95, 0x8a, 0x86, 0xfa, 0xf6, 0x4a, 0xad, + 0x83, 0xcb, 0xaa, 0xad, 0xce, 0x4c, 0xdc, 0x91, 0xde, 0xba, 0x1d, 0x79, 0x71, 0x79, 0x47, 0x6a, + 0x6b, 0xd8, 0x92, 0xb6, 0xe4, 0xc9, 0x8a, 0xf1, 0xcc, 0xaf, 0x34, 0xd7, 0x1e, 0x2d, 0x6b, 0xff, + 0x1e, 0xb2, 0xb2, 0x35, 0xbc, 0x0b, 0x9a, 0x54, 0x9c, 0xd9, 0x0e, 0x89, 0x42, 0x16, 0x00, 0xbe, + 0x03, 0x69, 0xdf, 0x73, 0x95, 0xd8, 0xad, 0xcd, 0x60, 0x56, 0x49, 0x9f, 0x74, 0xda, 0x56, 0x88, + 0x61, 0x0c, 0x99, 0x25, 0xa5, 0xe5, 0xb9, 0x39, 0x43, 0x90, 0x09, 0xbb, 0xc1, 0x7d, 0xd8, 0x4e, + 0x7a, 0x51, 0x78, 0x6f, 0xdd, 0x8b, 0x93, 0x92, 0x96, 0x1f, 0xdc, 0xec, 0x61, 0x1a, 0x29, 0x3c, + 0x82, 0xdb, 0xc9, 0xc2, 0xe0, 0xda, 0x7a, 0xe9, 0x14, 0x59, 0xfd, 0xa6, 0x1a, 0x1b, 0xa9, 0x56, + 0xeb, 0xfc, 0x42, 0x47, 0xbf, 0x2e, 0xf4, 0xd4, 0x97, 0x40, 0x47, 0xe7, 0x81, 0x8e, 0x7e, 0x04, + 0x3a, 0xfa, 0x1d, 0xe8, 0xe8, 0xeb, 0x1f, 0x3d, 0xf5, 0xe1, 0xfe, 0xe0, 0x50, 0x34, 0x3c, 0x6a, + 0x0e, 0xfc, 0x1e, 0x19, 0x92, 0x89, 0xc9, 0x06, 0x7d, 0xd3, 0x66, 0x9e, 0x30, 0x5d, 0x6e, 0x9b, + 0x11, 0xc9, 0x41, 0x2f, 0x27, 0x7f, 0xbd, 0xfd, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x91, 0xf3, + 0x7a, 0xa9, 0x3b, 0x05, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -698,11 +780,71 @@ func (m *NodePrepareResourceResponse) MarshalToSizedBuffer(dAtA []byte) (int, er i-- dAtA[i] = 0x12 } - if len(m.CDIDevices) > 0 { - for iNdEx := len(m.CDIDevices) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.CDIDevices[iNdEx]) - copy(dAtA[i:], m.CDIDevices[iNdEx]) - i = encodeVarintApi(dAtA, i, uint64(len(m.CDIDevices[iNdEx]))) + if len(m.Devices) > 0 { + for iNdEx := len(m.Devices) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Devices[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintApi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *Device) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Device) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Device) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.CDIDeviceIDs) > 0 { + for iNdEx := len(m.CDIDeviceIDs) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.CDIDeviceIDs[iNdEx]) + copy(dAtA[i:], m.CDIDeviceIDs[iNdEx]) + i = encodeVarintApi(dAtA, i, uint64(len(m.CDIDeviceIDs[iNdEx]))) + i-- + dAtA[i] = 0x22 + } + } + if len(m.DeviceName) > 0 { + i -= len(m.DeviceName) + copy(dAtA[i:], m.DeviceName) + i = encodeVarintApi(dAtA, i, uint64(len(m.DeviceName))) + i-- + dAtA[i] = 0x1a + } + if len(m.PoolName) > 0 { + i -= len(m.PoolName) + copy(dAtA[i:], m.PoolName) + i = encodeVarintApi(dAtA, i, uint64(len(m.PoolName))) + i-- + dAtA[i] = 0x12 + } + if len(m.RequestNames) > 0 { + for iNdEx := len(m.RequestNames) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RequestNames[iNdEx]) + copy(dAtA[i:], m.RequestNames[iNdEx]) + i = encodeVarintApi(dAtA, i, uint64(len(m.RequestNames[iNdEx]))) i-- dAtA[i] = 0xa } @@ -853,10 +995,10 @@ func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x1a } - if len(m.Uid) > 0 { - i -= len(m.Uid) - copy(dAtA[i:], m.Uid) - i = encodeVarintApi(dAtA, i, uint64(len(m.Uid))) + if len(m.UID) > 0 { + i -= len(m.UID) + copy(dAtA[i:], m.UID) + i = encodeVarintApi(dAtA, i, uint64(len(m.UID))) i-- dAtA[i] = 0x12 } @@ -924,9 +1066,9 @@ func (m *NodePrepareResourceResponse) Size() (n int) { } var l int _ = l - if len(m.CDIDevices) > 0 { - for _, s := range m.CDIDevices { - l = len(s) + if len(m.Devices) > 0 { + for _, e := range m.Devices { + l = e.Size() n += 1 + l + sovApi(uint64(l)) } } @@ -937,6 +1079,35 @@ func (m *NodePrepareResourceResponse) Size() (n int) { return n } +func (m *Device) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.RequestNames) > 0 { + for _, s := range m.RequestNames { + l = len(s) + n += 1 + l + sovApi(uint64(l)) + } + } + l = len(m.PoolName) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + l = len(m.DeviceName) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.CDIDeviceIDs) > 0 { + for _, s := range m.CDIDeviceIDs { + l = len(s) + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + func (m *NodeUnprepareResourcesRequest) Size() (n int) { if m == nil { return 0 @@ -997,7 +1168,7 @@ func (m *Claim) Size() (n int) { if l > 0 { n += 1 + l + sovApi(uint64(l)) } - l = len(m.Uid) + l = len(m.UID) if l > 0 { n += 1 + l + sovApi(uint64(l)) } @@ -1053,13 +1224,31 @@ func (this *NodePrepareResourceResponse) String() string { if this == nil { return "nil" } + repeatedStringForDevices := "[]*Device{" + for _, f := range this.Devices { + repeatedStringForDevices += strings.Replace(f.String(), "Device", "Device", 1) + "," + } + repeatedStringForDevices += "}" s := strings.Join([]string{`&NodePrepareResourceResponse{`, - `CDIDevices:` + fmt.Sprintf("%v", this.CDIDevices) + `,`, + `Devices:` + repeatedStringForDevices + `,`, `Error:` + fmt.Sprintf("%v", this.Error) + `,`, `}`, }, "") return s } +func (this *Device) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Device{`, + `RequestNames:` + fmt.Sprintf("%v", this.RequestNames) + `,`, + `PoolName:` + fmt.Sprintf("%v", this.PoolName) + `,`, + `DeviceName:` + fmt.Sprintf("%v", this.DeviceName) + `,`, + `CDIDeviceIDs:` + fmt.Sprintf("%v", this.CDIDeviceIDs) + `,`, + `}`, + }, "") + return s +} func (this *NodeUnprepareResourcesRequest) String() string { if this == nil { return "nil" @@ -1111,7 +1300,7 @@ func (this *Claim) String() string { } s := strings.Join([]string{`&Claim{`, `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, - `Uid:` + fmt.Sprintf("%v", this.Uid) + `,`, + `UID:` + fmt.Sprintf("%v", this.UID) + `,`, `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `}`, }, "") @@ -1419,9 +1608,9 @@ func (m *NodePrepareResourceResponse) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field CDIDevices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Devices", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowApi @@ -1431,23 +1620,25 @@ func (m *NodePrepareResourceResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthApi } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthApi } if postIndex > l { return io.ErrUnexpectedEOF } - m.CDIDevices = append(m.CDIDevices, string(dAtA[iNdEx:postIndex])) + m.Devices = append(m.Devices, &Device{}) + if err := m.Devices[len(m.Devices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 2: if wireType != 2 { @@ -1502,6 +1693,184 @@ func (m *NodePrepareResourceResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *Device) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Device: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Device: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RequestNames", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RequestNames = append(m.RequestNames, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PoolName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PoolName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DeviceName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DeviceName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CDIDeviceIDs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CDIDeviceIDs = append(m.CDIDeviceIDs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NodeUnprepareResourcesRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1910,7 +2279,7 @@ func (m *Claim) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Uid", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field UID", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -1938,7 +2307,7 @@ func (m *Claim) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Uid = string(dAtA[iNdEx:postIndex]) + m.UID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { diff --git a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.proto b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.proto index 04a374535b6..ddc33490252 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.proto +++ b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.proto @@ -62,14 +62,31 @@ message NodePrepareResourcesResponse { message NodePrepareResourceResponse { // These are the additional devices that kubelet must - // make available via the container runtime. A resource + // make available via the container runtime. A claim + // may have zero or more requests and each request // may have zero or more devices. - repeated string cdi_devices = 1 [(gogoproto.customname) = "CDIDevices"]; + repeated Device devices = 1; // If non-empty, preparing the ResourceClaim failed. // cdi_devices is ignored in that case. string error = 2; } +message Device { + // The requests in the claim that this device is associated with. + // Optional. If empty, the device is associated with all requests. + repeated string request_names = 1; + + // The pool which contains the device. Required. + string pool_name = 2; + + // The device itself. Required. + string device_name = 3; + + // A single device instance may map to several CDI device IDs. + // None is also valid. + repeated string cdi_device_ids = 4 [(gogoproto.customname) = "CDIDeviceIDs"]; +} + message NodeUnprepareResourcesRequest { // The list of ResourceClaims that are to be unprepared. repeated Claim claims = 1; @@ -93,7 +110,7 @@ message Claim { string namespace = 1; // The UID of the Resource claim (ResourceClaim.meta.UUID). // This field is REQUIRED. - string uid = 2; + string uid = 2 [(gogoproto.customname) = "UID"]; // The name of the Resource claim (ResourceClaim.meta.Name) // This field is REQUIRED. string name = 3; diff --git a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go index 5ca454588ce..a67c289a50a 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.pb.go @@ -586,7 +586,8 @@ func (m *NUMANode) GetID() int64 { // DynamicResource contains information about the devices assigned to a container by DRA type DynamicResource struct { - ClassName string `protobuf:"bytes,1,opt,name=class_name,json=className,proto3" json:"class_name,omitempty"` + // tombstone: removed in 1.31 because claims are no longer associated with one class + // string class_name = 1; ClaimName string `protobuf:"bytes,2,opt,name=claim_name,json=claimName,proto3" json:"claim_name,omitempty"` ClaimNamespace string `protobuf:"bytes,3,opt,name=claim_namespace,json=claimNamespace,proto3" json:"claim_namespace,omitempty"` ClaimResources []*ClaimResource `protobuf:"bytes,4,rep,name=claim_resources,json=claimResources,proto3" json:"claim_resources,omitempty"` @@ -626,13 +627,6 @@ func (m *DynamicResource) XXX_DiscardUnknown() { var xxx_messageInfo_DynamicResource proto.InternalMessageInfo -func (m *DynamicResource) GetClassName() string { - if m != nil { - return m.ClassName - } - return "" -} - func (m *DynamicResource) GetClaimName() string { if m != nil { return m.ClaimName @@ -654,9 +648,15 @@ func (m *DynamicResource) GetClaimResources() []*ClaimResource { return nil } -// ClaimResource contains per plugin resource information +// ClaimResource contains resource information. The driver name/pool name/device name +// triplet uniquely identifies the device. Should DRA get extended to other kinds +// of resources, then device_name will be empty and other fields will get added. +// Each device at the DRA API level may map to zero or more CDI devices. type ClaimResource struct { CDIDevices []*CDIDevice `protobuf:"bytes,1,rep,name=cdi_devices,json=cdiDevices,proto3" json:"cdi_devices,omitempty"` + DriverName string `protobuf:"bytes,2,opt,name=driver_name,json=driverName,proto3" json:"driver_name,omitempty"` + PoolName string `protobuf:"bytes,3,opt,name=pool_name,json=poolName,proto3" json:"pool_name,omitempty"` + DeviceName string `protobuf:"bytes,4,opt,name=device_name,json=deviceName,proto3" json:"device_name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_sizecache int32 `json:"-"` } @@ -700,6 +700,27 @@ func (m *ClaimResource) GetCDIDevices() []*CDIDevice { return nil } +func (m *ClaimResource) GetDriverName() string { + if m != nil { + return m.DriverName + } + return "" +} + +func (m *ClaimResource) GetPoolName() string { + if m != nil { + return m.PoolName + } + return "" +} + +func (m *ClaimResource) GetDeviceName() string { + if m != nil { + return m.DeviceName + } + return "" +} + // CDIDevice specifies a CDI device information type CDIDevice struct { // Fully qualified CDI device name @@ -871,55 +892,57 @@ func init() { func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } var fileDescriptor_00212fb1f9d3bf1c = []byte{ - // 760 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x5d, 0x6f, 0x12, 0x4d, - 0x14, 0xee, 0xb0, 0xf4, 0x83, 0x53, 0xe8, 0xc7, 0xbc, 0x6f, 0x5a, 0x4a, 0x5b, 0x20, 0xdb, 0x8b, - 0x36, 0x51, 0x21, 0xad, 0xd1, 0x18, 0x2f, 0x4c, 0x3f, 0x30, 0x0d, 0x89, 0xad, 0x75, 0x53, 0x13, - 0xe3, 0x85, 0x64, 0xd9, 0x9d, 0xe2, 0xa4, 0xc0, 0x8c, 0xcc, 0x42, 0xc4, 0x2b, 0x2f, 0xfc, 0x01, - 0x5e, 0xf8, 0x53, 0xfc, 0x11, 0xbd, 0xf4, 0xd2, 0x2b, 0xd3, 0xe2, 0xcf, 0xf0, 0xc6, 0xcc, 0x0c, - 0xbb, 0x2c, 0xb0, 0xd8, 0xf4, 0x8a, 0x99, 0xf3, 0x3c, 0xe7, 0x70, 0xce, 0x73, 0xce, 0x9c, 0x85, - 0x84, 0xcd, 0x69, 0x81, 0xb7, 0x98, 0xc7, 0x70, 0xac, 0xb3, 0x9b, 0x79, 0x50, 0xa3, 0xde, 0xfb, - 0x76, 0xb5, 0xe0, 0xb0, 0x46, 0xb1, 0xc6, 0x6a, 0xac, 0xa8, 0xa0, 0x6a, 0xfb, 0x42, 0xdd, 0xd4, - 0x45, 0x9d, 0xb4, 0x8b, 0xb9, 0x09, 0xeb, 0x07, 0xf5, 0x3a, 0x73, 0x6c, 0xcf, 0xae, 0xd6, 0x89, - 0x45, 0x04, 0x6b, 0xb7, 0x1c, 0x22, 0x2c, 0xf2, 0xa1, 0x4d, 0x84, 0x67, 0x7e, 0x43, 0xb0, 0x11, - 0x8d, 0x0b, 0xce, 0x9a, 0x82, 0xe0, 0x02, 0xcc, 0xba, 0xa4, 0x43, 0x1d, 0x22, 0xd2, 0x28, 0x6f, - 0xec, 0xcc, 0xef, 0xfd, 0x5f, 0xe8, 0xec, 0x16, 0x8e, 0x58, 0xd3, 0xb3, 0x69, 0x93, 0xb4, 0x4a, - 0x1a, 0xb3, 0x7c, 0x12, 0x5e, 0x85, 0x59, 0x87, 0xb7, 0x2b, 0xd4, 0x15, 0xe9, 0x58, 0xde, 0xd8, - 0x31, 0xac, 0x19, 0x87, 0xb7, 0xcb, 0xae, 0xc0, 0xf7, 0x60, 0xa6, 0x41, 0x1a, 0xac, 0xd5, 0x4d, - 0x1b, 0x2a, 0xce, 0x7f, 0x43, 0x71, 0x4e, 0x14, 0x64, 0xf5, 0x29, 0xe6, 0x1a, 0xac, 0xbe, 0xa0, - 0xc2, 0x3b, 0x63, 0xee, 0x58, 0xc6, 0xaf, 0x20, 0x3d, 0x0e, 0xf5, 0x93, 0x7d, 0x04, 0x29, 0xce, - 0xdc, 0x4a, 0xcb, 0x07, 0xfa, 0x29, 0x2f, 0xc9, 0xbf, 0x1a, 0x72, 0x48, 0xf2, 0xd0, 0xcd, 0xfc, - 0x08, 0xc9, 0x30, 0x8a, 0x31, 0xc4, 0x9b, 0x76, 0x83, 0xa4, 0x51, 0x1e, 0xed, 0x24, 0x2c, 0x75, - 0xc6, 0x1b, 0x90, 0x90, 0xbf, 0x82, 0xdb, 0x0e, 0x49, 0xc7, 0x14, 0x30, 0x30, 0xe0, 0xc7, 0x00, - 0x8e, 0x5f, 0x8a, 0xe8, 0x17, 0xb8, 0x32, 0x54, 0xe0, 0xe0, 0xbf, 0x43, 0x4c, 0xf3, 0x1a, 0x01, - 0x1e, 0xa7, 0x44, 0x26, 0x10, 0x6a, 0x44, 0xec, 0x8e, 0x8d, 0x30, 0x26, 0x34, 0x22, 0x7e, 0x6b, - 0x23, 0xf0, 0x3e, 0x2c, 0xbb, 0xdd, 0xa6, 0xdd, 0xa0, 0x4e, 0x48, 0xd5, 0xe9, 0x81, 0x5f, 0x49, - 0x83, 0x7e, 0xea, 0xd6, 0x92, 0x3b, 0x6c, 0x10, 0xa6, 0x07, 0x8b, 0x23, 0xc1, 0x71, 0x0e, 0xe6, - 0x75, 0xf8, 0x8a, 0xd7, 0xe5, 0x7e, 0x95, 0xa0, 0x4d, 0xe7, 0x5d, 0x4e, 0x64, 0xfd, 0x82, 0x7e, - 0xd2, 0x3a, 0xc7, 0x2d, 0x75, 0xc6, 0xf7, 0x61, 0xce, 0x63, 0x9c, 0xd5, 0x59, 0x4d, 0x4e, 0x10, - 0xf2, 0xdb, 0x7a, 0xde, 0xb7, 0x95, 0x9b, 0x17, 0xcc, 0x0a, 0x18, 0xe6, 0x17, 0x04, 0x4b, 0xa3, - 0xda, 0xe0, 0x2d, 0x48, 0xf9, 0x45, 0x54, 0x42, 0xfa, 0x26, 0x7d, 0xe3, 0xa9, 0xd4, 0x79, 0x13, - 0x40, 0x4b, 0x18, 0xcc, 0x70, 0xc2, 0x4a, 0x68, 0x8b, 0x54, 0xef, 0x6e, 0x69, 0xec, 0x41, 0x32, - 0x8c, 0x60, 0x13, 0xa6, 0x9b, 0xcc, 0x0d, 0x06, 0x33, 0x29, 0x5d, 0x4f, 0x5f, 0x9f, 0x1c, 0x9c, - 0x32, 0x97, 0x58, 0x1a, 0x32, 0x33, 0x30, 0xe7, 0x9b, 0xf0, 0x02, 0xc4, 0xca, 0x25, 0x95, 0xa6, - 0x61, 0xc5, 0xca, 0x25, 0xf3, 0x3b, 0x82, 0xc5, 0x11, 0xc9, 0x65, 0xc2, 0x4e, 0xdd, 0x16, 0x22, - 0x5c, 0x52, 0x42, 0x59, 0xfc, 0x7a, 0x9c, 0xba, 0x4d, 0x1b, 0x1a, 0x8e, 0x05, 0x30, 0x6d, 0x28, - 0x78, 0x1b, 0x16, 0x07, 0xb0, 0x9e, 0x6e, 0x43, 0x71, 0x16, 0x02, 0x8e, 0x1e, 0xf1, 0xa7, 0x3e, - 0x71, 0x30, 0x07, 0x7a, 0x7e, 0x96, 0xd5, 0xfc, 0x48, 0x28, 0x98, 0x02, 0xed, 0x3b, 0x98, 0x81, - 0x97, 0x90, 0x1a, 0x22, 0xe0, 0x67, 0x30, 0xef, 0xb8, 0xb4, 0x32, 0xbc, 0x59, 0x52, 0x2a, 0x50, - 0xa9, 0xac, 0xdb, 0x75, 0xb8, 0xd0, 0xfb, 0x95, 0x83, 0xe0, 0x2a, 0xdf, 0x8d, 0x4b, 0xfb, 0x67, - 0x33, 0x07, 0x89, 0x00, 0x89, 0x7a, 0x2d, 0xe6, 0x1b, 0x58, 0x39, 0x26, 0x51, 0xfb, 0x03, 0xaf, - 0xc1, 0x9c, 0xdc, 0x11, 0x21, 0x8f, 0x59, 0xce, 0x5c, 0xa5, 0xc5, 0x96, 0x5e, 0x1f, 0xa3, 0xef, - 0x3c, 0xd9, 0xc7, 0x95, 0xcd, 0x3c, 0x83, 0xd5, 0xb1, 0xc8, 0x93, 0xd7, 0x0f, 0xba, 0x7d, 0xfd, - 0xec, 0xfd, 0x41, 0x80, 0xc3, 0xb0, 0x5c, 0x6f, 0xa4, 0x85, 0x8f, 0x20, 0x2e, 0x4f, 0x78, 0x5d, - 0xba, 0x4f, 0xd8, 0x86, 0x99, 0x8d, 0x68, 0x50, 0x27, 0x64, 0x4e, 0xe1, 0x77, 0x2a, 0xdb, 0xa8, - 0x0d, 0x8f, 0x73, 0xd2, 0xf5, 0x1f, 0xdf, 0x86, 0x4c, 0x7e, 0x32, 0x21, 0x88, 0xbf, 0x0f, 0xc6, - 0x31, 0xf1, 0x70, 0x46, 0x52, 0xa3, 0x05, 0xcf, 0xac, 0x47, 0x62, 0x7e, 0x84, 0xc3, 0xe7, 0x57, - 0x37, 0x59, 0xf4, 0xf3, 0x26, 0x3b, 0xf5, 0xb9, 0x97, 0x45, 0x57, 0xbd, 0x2c, 0xfa, 0xd1, 0xcb, - 0xa2, 0xeb, 0x5e, 0x16, 0x7d, 0xfd, 0x9d, 0x9d, 0x7a, 0xbb, 0x7d, 0xf9, 0x44, 0x14, 0x28, 0x2b, - 0x5e, 0xb6, 0xab, 0xa4, 0x4e, 0xbc, 0x22, 0xbf, 0xac, 0x15, 0x6d, 0x4e, 0x45, 0x91, 0x33, 0x37, - 0xd0, 0xb9, 0xd8, 0xd9, 0xad, 0xce, 0xa8, 0xcf, 0xdd, 0xc3, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, - 0x49, 0xac, 0x87, 0x00, 0x2e, 0x07, 0x00, 0x00, + // 789 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x4d, 0x6f, 0xda, 0x48, + 0x18, 0xce, 0x60, 0x92, 0xc0, 0x0b, 0xe4, 0x63, 0x76, 0x95, 0x10, 0x48, 0x00, 0x39, 0x87, 0x44, + 0xda, 0x5d, 0x50, 0xb2, 0xda, 0xd5, 0x6a, 0x0f, 0xab, 0x7c, 0xb0, 0x8a, 0x90, 0x36, 0x51, 0xd6, + 0x4a, 0xa5, 0xaa, 0x87, 0x22, 0x63, 0x4f, 0xa8, 0x15, 0x60, 0xa6, 0x1e, 0x83, 0x4a, 0x4f, 0x3d, + 0xf4, 0x07, 0xf4, 0xd0, 0xfe, 0x8d, 0xfe, 0x8e, 0x1c, 0x7b, 0xec, 0xa9, 0x4a, 0xe8, 0xcf, 0xe8, + 0xa5, 0x9a, 0x19, 0xdb, 0x18, 0x30, 0x8d, 0x72, 0x62, 0xe6, 0x79, 0x9e, 0xf7, 0x9d, 0xf7, 0x8b, + 0xd7, 0x90, 0x36, 0x99, 0x53, 0x65, 0x2e, 0xf5, 0x28, 0x4e, 0x0c, 0x0e, 0x0a, 0xbf, 0xb5, 0x1d, + 0xef, 0x45, 0xbf, 0x55, 0xb5, 0x68, 0xb7, 0xd6, 0xa6, 0x6d, 0x5a, 0x93, 0x54, 0xab, 0x7f, 0x2d, + 0x6f, 0xf2, 0x22, 0x4f, 0xca, 0x44, 0xdf, 0x81, 0xe2, 0x71, 0xa7, 0x43, 0x2d, 0xd3, 0x33, 0x5b, + 0x1d, 0x62, 0x10, 0x4e, 0xfb, 0xae, 0x45, 0xb8, 0x41, 0x5e, 0xf6, 0x09, 0xf7, 0xf4, 0xf7, 0x08, + 0xb6, 0xe3, 0x79, 0xce, 0x68, 0x8f, 0x13, 0x5c, 0x85, 0x65, 0x9b, 0x0c, 0x1c, 0x8b, 0xf0, 0x3c, + 0xaa, 0x68, 0xfb, 0x99, 0xc3, 0x9f, 0xab, 0x83, 0x83, 0xea, 0x29, 0xed, 0x79, 0xa6, 0xd3, 0x23, + 0x6e, 0x5d, 0x71, 0x46, 0x20, 0xc2, 0x9b, 0xb0, 0x6c, 0xb1, 0x7e, 0xd3, 0xb1, 0x79, 0x3e, 0x51, + 0xd1, 0xf6, 0x35, 0x63, 0xc9, 0x62, 0xfd, 0x86, 0xcd, 0xf1, 0x2f, 0xb0, 0xd4, 0x25, 0x5d, 0xea, + 0x0e, 0xf3, 0x9a, 0xf4, 0xf3, 0xd3, 0x84, 0x9f, 0x73, 0x49, 0x19, 0xbe, 0x44, 0xdf, 0x82, 0xcd, + 0xff, 0x1c, 0xee, 0x5d, 0x52, 0x7b, 0x26, 0xe2, 0xff, 0x21, 0x3f, 0x4b, 0xf9, 0xc1, 0xfe, 0x01, + 0x39, 0x46, 0xed, 0xa6, 0x1b, 0x10, 0x7e, 0xc8, 0x6b, 0xe2, 0xa9, 0x09, 0x83, 0x2c, 0x8b, 0xdc, + 0xf4, 0x57, 0x90, 0x8d, 0xb2, 0x18, 0x43, 0xb2, 0x67, 0x76, 0x49, 0x1e, 0x55, 0xd0, 0x7e, 0xda, + 0x90, 0x67, 0xbc, 0x0d, 0x69, 0xf1, 0xcb, 0x99, 0x69, 0x91, 0x7c, 0x42, 0x12, 0x63, 0x00, 0xff, + 0x09, 0x60, 0x05, 0xa9, 0x70, 0x3f, 0xc1, 0x8d, 0x89, 0x04, 0xc7, 0x6f, 0x47, 0x94, 0xfa, 0x1d, + 0x02, 0x3c, 0x2b, 0x89, 0x0d, 0x20, 0xd2, 0x88, 0xc4, 0x23, 0x1b, 0xa1, 0xcd, 0x69, 0x44, 0xf2, + 0xc1, 0x46, 0xe0, 0x23, 0x58, 0xb7, 0x87, 0x3d, 0xb3, 0xeb, 0x58, 0x91, 0xaa, 0x2e, 0x8e, 0xed, + 0xea, 0x8a, 0x0c, 0x42, 0x37, 0xd6, 0xec, 0x49, 0x80, 0xeb, 0x1e, 0xac, 0x4e, 0x39, 0xc7, 0x65, + 0xc8, 0x28, 0xf7, 0x4d, 0x6f, 0xc8, 0x82, 0x2c, 0x41, 0x41, 0x57, 0x43, 0x46, 0x44, 0xfe, 0xdc, + 0x79, 0xad, 0xea, 0x9c, 0x34, 0xe4, 0x19, 0xff, 0x0a, 0x29, 0x8f, 0x32, 0xda, 0xa1, 0x6d, 0x31, + 0x41, 0x28, 0x68, 0xeb, 0x95, 0x8f, 0x35, 0x7a, 0xd7, 0xd4, 0x08, 0x15, 0xfa, 0x5b, 0x04, 0x6b, + 0xd3, 0xb5, 0xc1, 0xbb, 0x90, 0x0b, 0x92, 0x68, 0x46, 0xea, 0x9b, 0x0d, 0xc0, 0x0b, 0x51, 0xe7, + 0x1d, 0x00, 0x55, 0xc2, 0x70, 0x86, 0xd3, 0x46, 0x5a, 0x21, 0xa2, 0x7a, 0x8f, 0x0b, 0xe3, 0x10, + 0xb2, 0x51, 0x06, 0xeb, 0xb0, 0xd8, 0xa3, 0x76, 0x38, 0x98, 0x59, 0x61, 0x7a, 0xf1, 0xe4, 0xfc, + 0xf8, 0x82, 0xda, 0xc4, 0x50, 0x94, 0x5e, 0x80, 0x54, 0x00, 0xe1, 0x15, 0x48, 0x34, 0xea, 0x32, + 0x4c, 0xcd, 0x48, 0x34, 0xea, 0xfa, 0x07, 0x04, 0xab, 0x53, 0x25, 0x17, 0x01, 0x5b, 0x1d, 0xd3, + 0xe9, 0xaa, 0x94, 0xfc, 0xd1, 0x94, 0x88, 0xcc, 0x67, 0x0f, 0x56, 0xc7, 0xb4, 0x1a, 0x5f, 0x4d, + 0x6a, 0x56, 0x42, 0x8d, 0x9a, 0xe1, 0xbf, 0x03, 0xe1, 0xb8, 0xd1, 0x6a, 0x40, 0xd6, 0xe5, 0x80, + 0x08, 0x2a, 0x6c, 0xb3, 0xb2, 0x1d, 0x37, 0xf9, 0x23, 0x82, 0xdc, 0x84, 0x02, 0xff, 0x03, 0x19, + 0xcb, 0x76, 0x9a, 0x93, 0xbb, 0x23, 0x27, 0x3d, 0xd5, 0x1b, 0xaa, 0x21, 0x27, 0x2b, 0xa3, 0x2f, + 0x65, 0x08, 0xaf, 0xe2, 0x9f, 0x61, 0x3b, 0x41, 0xaf, 0xca, 0x90, 0xb1, 0x5d, 0x67, 0x40, 0xdc, + 0x68, 0x5a, 0xa0, 0x20, 0x99, 0x57, 0x11, 0xd2, 0x8c, 0xd2, 0x8e, 0xa2, 0x55, 0x46, 0x29, 0x01, + 0x48, 0x52, 0x58, 0xab, 0x26, 0x4a, 0x3a, 0xe9, 0x5b, 0x4b, 0x48, 0x08, 0xf4, 0x32, 0xa4, 0xc3, + 0x87, 0xe3, 0xfe, 0x6e, 0xfa, 0x53, 0xd8, 0x38, 0x23, 0x71, 0x0b, 0x08, 0x6f, 0x41, 0x4a, 0x2c, + 0x99, 0x88, 0xc5, 0x32, 0xa3, 0xb6, 0x7c, 0x76, 0x57, 0xed, 0x9f, 0xe9, 0x45, 0x91, 0xf5, 0x79, + 0x89, 0xe9, 0x97, 0xb0, 0x39, 0xe3, 0x79, 0xfe, 0xfe, 0x42, 0x0f, 0xef, 0xaf, 0xc3, 0x6f, 0x08, + 0x70, 0x94, 0x16, 0xfb, 0x91, 0xb8, 0xf8, 0x14, 0x92, 0xe2, 0x84, 0x8b, 0xc2, 0x7c, 0xce, 0x3a, + 0x2d, 0x6c, 0xc7, 0x93, 0x2a, 0x20, 0x7d, 0x01, 0x3f, 0x97, 0xd1, 0xc6, 0x7d, 0x22, 0x70, 0x59, + 0x98, 0xfe, 0xe0, 0xe3, 0x52, 0xa8, 0xcc, 0x17, 0x84, 0xfe, 0x8f, 0x40, 0x3b, 0x23, 0x1e, 0x2e, + 0x08, 0x69, 0x7c, 0xc1, 0x0b, 0xc5, 0x58, 0x2e, 0xf0, 0x70, 0xf2, 0xef, 0xed, 0x7d, 0x09, 0x7d, + 0xbe, 0x2f, 0x2d, 0xbc, 0x19, 0x95, 0xd0, 0xed, 0xa8, 0x84, 0x3e, 0x8d, 0x4a, 0xe8, 0x6e, 0x54, + 0x42, 0xef, 0xbe, 0x96, 0x16, 0x9e, 0xed, 0xdd, 0xfc, 0xc5, 0xab, 0x0e, 0xad, 0xdd, 0xf4, 0x5b, + 0xa4, 0x43, 0xbc, 0x1a, 0xbb, 0x69, 0xd7, 0x4c, 0xe6, 0xf0, 0x1a, 0xa3, 0x76, 0x58, 0xe7, 0xda, + 0xe0, 0xa0, 0xb5, 0x24, 0xbf, 0x97, 0xbf, 0x7f, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x8c, 0x1b, 0x18, + 0xf9, 0x6f, 0x07, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1580,13 +1603,6 @@ func (m *DynamicResource) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x12 } - if len(m.ClassName) > 0 { - i -= len(m.ClassName) - copy(dAtA[i:], m.ClassName) - i = encodeVarintApi(dAtA, i, uint64(len(m.ClassName))) - i-- - dAtA[i] = 0xa - } return len(dAtA) - i, nil } @@ -1610,6 +1626,27 @@ func (m *ClaimResource) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.DeviceName) > 0 { + i -= len(m.DeviceName) + copy(dAtA[i:], m.DeviceName) + i = encodeVarintApi(dAtA, i, uint64(len(m.DeviceName))) + i-- + dAtA[i] = 0x22 + } + if len(m.PoolName) > 0 { + i -= len(m.PoolName) + copy(dAtA[i:], m.PoolName) + i = encodeVarintApi(dAtA, i, uint64(len(m.PoolName))) + i-- + dAtA[i] = 0x1a + } + if len(m.DriverName) > 0 { + i -= len(m.DriverName) + copy(dAtA[i:], m.DriverName) + i = encodeVarintApi(dAtA, i, uint64(len(m.DriverName))) + i-- + dAtA[i] = 0x12 + } if len(m.CDIDevices) > 0 { for iNdEx := len(m.CDIDevices) - 1; iNdEx >= 0; iNdEx-- { { @@ -1938,10 +1975,6 @@ func (m *DynamicResource) Size() (n int) { } var l int _ = l - l = len(m.ClassName) - if l > 0 { - n += 1 + l + sovApi(uint64(l)) - } l = len(m.ClaimName) if l > 0 { n += 1 + l + sovApi(uint64(l)) @@ -1971,6 +2004,18 @@ func (m *ClaimResource) Size() (n int) { n += 1 + l + sovApi(uint64(l)) } } + l = len(m.DriverName) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + l = len(m.PoolName) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + l = len(m.DeviceName) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } return n } @@ -2183,7 +2228,6 @@ func (this *DynamicResource) String() string { } repeatedStringForClaimResources += "}" s := strings.Join([]string{`&DynamicResource{`, - `ClassName:` + fmt.Sprintf("%v", this.ClassName) + `,`, `ClaimName:` + fmt.Sprintf("%v", this.ClaimName) + `,`, `ClaimNamespace:` + fmt.Sprintf("%v", this.ClaimNamespace) + `,`, `ClaimResources:` + repeatedStringForClaimResources + `,`, @@ -2202,6 +2246,9 @@ func (this *ClaimResource) String() string { repeatedStringForCDIDevices += "}" s := strings.Join([]string{`&ClaimResource{`, `CDIDevices:` + repeatedStringForCDIDevices + `,`, + `DriverName:` + fmt.Sprintf("%v", this.DriverName) + `,`, + `PoolName:` + fmt.Sprintf("%v", this.PoolName) + `,`, + `DeviceName:` + fmt.Sprintf("%v", this.DeviceName) + `,`, `}`, }, "") return s @@ -3500,38 +3547,6 @@ func (m *DynamicResource) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: DynamicResource: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ClassName", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowApi - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthApi - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthApi - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ClassName = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClaimName", wireType) @@ -3714,6 +3729,102 @@ func (m *ClaimResource) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DriverName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DriverName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PoolName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PoolName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DeviceName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthApi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DeviceName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipApi(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto index 9e6022ac32c..93ab0185d58 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto +++ b/staging/src/k8s.io/kubelet/pkg/apis/podresources/v1/api.proto @@ -82,15 +82,22 @@ message NUMANode { // DynamicResource contains information about the devices assigned to a container by DRA message DynamicResource { - string class_name = 1; + // tombstone: removed in 1.31 because claims are no longer associated with one class + // string class_name = 1; string claim_name = 2; string claim_namespace = 3; repeated ClaimResource claim_resources = 4; } -// ClaimResource contains per plugin resource information +// ClaimResource contains resource information. The driver name/pool name/device name +// triplet uniquely identifies the device. Should DRA get extended to other kinds +// of resources, then device_name will be empty and other fields will get added. +// Each device at the DRA API level may map to zero or more CDI devices. message ClaimResource { repeated CDIDevice cdi_devices = 1 [(gogoproto.customname) = "CDIDevices"]; + string driver_name = 2; + string pool_name = 3; + string device_name = 4; } // CDIDevice specifies a CDI device information