Add CDI devices to device plugin API

This change adds CDI device IDs to the ContainerAllocateResponse in the
device plugin API. This allows a device plugin to specify CDI devices
by their unique fully-qualified CDI device names using the related field
in the CRI specification.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

pkg/features/kube_features.go

@@ -221,6 +221,13 @@ const (
 	// (e.g. in a Deployment), which is the historical default.
 	DefaultHostNetworkHostPortsInPodTemplates featuregate.Feature = "DefaultHostNetworkHostPortsInPodTemplates"
 
+	// owner: @elezar
+	// kep: http://kep.k8s.io/4009
+	// alpha: v1.28
+	//
+	// Add support for CDI Device IDs in the Device Plugin API.
+	DevicePluginCDIDevices featuregate.Feature = "DevicePluginCDIDevices"
+
 	// owner: @andrewsykim
 	// alpha: v1.22
 	//
@@ -908,6 +915,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	DisableKubeletCloudCredentialProviders: {Default: false, PreRelease: featuregate.Alpha},
 
+	DevicePluginCDIDevices: {Default: false, PreRelease: featuregate.Alpha},
+
 	DownwardAPIHugePages: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in v1.29
 
 	DynamicResourceAllocation: {Default: false, PreRelease: featuregate.Alpha},

pkg/kubelet/cm/container_manager_linux.go

@@ -673,6 +673,7 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe
 	opts.Mounts = append(opts.Mounts, devOpts.Mounts...)
 	opts.Envs = append(opts.Envs, devOpts.Envs...)
 	opts.Annotations = append(opts.Annotations, devOpts.Annotations...)
+	opts.CDIDevices = append(opts.CDIDevices, devOpts.CDIDevices...)
 	return opts, nil
 }
 

pkg/kubelet/cm/devicemanager/manager_test.go

@@ -576,6 +576,7 @@ type containerAllocateResponseBuilder struct {
 	devices    map[string]string
 	mounts     map[string]string
 	envs       map[string]string
+	cdiDevices []string
 }
 
 // containerAllocateResponseBuilderOption defines a functional option for a containerAllocateResponseBuilder
@@ -602,6 +603,13 @@ func withEnvs(envs map[string]string) containerAllocateResponseBuilderOption {
 	}
 }
 
+// withCDIDevices sets the cdiDevices for the containerAllocateResponseBuilder
+func withCDIDevices(cdiDevices ...string) containerAllocateResponseBuilderOption {
+	return func(b *containerAllocateResponseBuilder) {
+		b.cdiDevices = cdiDevices
+	}
+}
+
 // newContainerAllocateResponse creates a ContainerAllocateResponse with the given options.
 func newContainerAllocateResponse(opts ...containerAllocateResponseBuilderOption) *pluginapi.ContainerAllocateResponse {
 	b := &containerAllocateResponseBuilder{}
@@ -633,6 +641,16 @@ func (b *containerAllocateResponseBuilder) Build() *pluginapi.ContainerAllocateR
 	for k, v := range b.envs {
 		resp.Envs[k] = v
 	}
+
+	var cdiDevices []*pluginapi.CDIDevice
+	for _, dev := range b.cdiDevices {
+		cdiDevice := pluginapi.CDIDevice{
+			Name: dev,
+		}
+		cdiDevices = append(cdiDevices, &cdiDevice)
+	}
+	resp.CDIDevices = cdiDevices
+
 	return resp
 }
 
@@ -660,6 +678,7 @@ func TestCheckpoint(t *testing.T) {
 		newContainerAllocateResponse(
 			withDevices(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}),
 			withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}),
+			withCDIDevices("domain1.com/resource1=dev1", "domain1.com/resource1=dev2"),
 		),
 	)
 	testManager.podDevices.insert("pod1", "con1", resourceName2,

pkg/kubelet/cm/devicemanager/pod_devices.go

@@ -21,9 +21,13 @@ import (
 
 	"k8s.io/klog/v2"
 
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
+	"k8s.io/kubernetes/pkg/kubelet/cm/util/cdi"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 )
 
@@ -244,6 +248,8 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi
 	mountsMap := make(map[string]string)
 	envsMap := make(map[string]string)
 	annotationsMap := make(map[string]string)
+	// Keep track of all CDI devices requested for the container.
+	allCDIDevices := sets.New[string]()
 	// Loops through AllocationResponses of all cached device resources.
 	for _, devices := range resources {
 		resp := devices.allocResp
@@ -252,6 +258,7 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi
 		// Mount points
 		// Device files
 		// Container annotations
+		// CDI device IDs
 		// These artifacts are per resource per container.
 		// Updates RunContainerOptions.Envs.
 		for k, v := range resp.Envs {
@@ -321,10 +328,78 @@ func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *Devi
 			annotationsMap[k] = v
 			opts.Annotations = append(opts.Annotations, kubecontainer.Annotation{Name: k, Value: v})
 		}
+
+		if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DevicePluginCDIDevices) {
+			// Updates for CDI devices.
+			cdiDevices := getCDIDeviceInfo(resp, allCDIDevices)
+			opts.CDIDevices = append(opts.CDIDevices, cdiDevices...)
 		}
+	}
+
+	// Although the CDI devices are expected to be empty when this feature is disabled, we still
+	// guard this with a feature gate to avoid any potential issues.
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DevicePluginCDIDevices) {
+		// We construct a resource ID from the pod UID and container name.
+		// This ID has no semantic meaning, and is only used to ensure that the generated CDI annotation key is unique
+		// for a given container. Since this is only called once per pod-container combination, this should be the case.
+		resourceID := podUID + "-" + contName
+		cdiAnnotations := getCDIAnnotations(resourceID, allCDIDevices, annotationsMap)
+		opts.Annotations = append(opts.Annotations, cdiAnnotations...)
+	}
+
 	return opts
 }
 
+// getCDIAnnotations returns the cdi annotations for a given container.
+// This creates a CDI annotation with a key of the form: devicemanager_{{resourceID}}.
+// The value of the annotation is a comma separated list of sorted CDI device IDs.
+// If the annotation key is already defined in the provided annotations map, then the existing value is used.
+func getCDIAnnotations(resourceID string, cdiDevices sets.Set[string], annotationsMap map[string]string) []kubecontainer.Annotation {
+	// We sort the CDI devices to ensure that the annotation value is deterministic.
+	sortedCDIDevices := sets.List[string](cdiDevices)
+	annotations, err := cdi.GenerateAnnotations(types.UID(resourceID), "devicemanager", sortedCDIDevices)
+	if err != nil {
+		klog.ErrorS(err, "Failed to create CDI annotations")
+		return nil
+	}
+
+	var cdiAnnotations []kubecontainer.Annotation
+	for _, annotation := range annotations {
+		if e, ok := annotationsMap[annotation.Name]; ok {
+			klog.V(4).InfoS("Skip existing annotation", "annotationKey", annotation.Name, "annotationValue", annotation.Value)
+			if e != annotation.Value {
+				klog.ErrorS(nil, "Annotation has conflicting setting", "annotationKey", annotation.Name, "expected", e, "got", annotation.Value)
+			}
+			continue
+		}
+		klog.V(4).InfoS("Add annotation", "annotationKey", annotation.Name, "annotationValue", annotation.Value)
+		annotationsMap[annotation.Name] = annotation.Value
+		cdiAnnotations = append(cdiAnnotations, kubecontainer.Annotation{Name: annotation.Name, Value: annotation.Value})
+	}
+
+	return cdiAnnotations
+}
+
+// getCDIDeviceInfo returns CDI devices from an allocate response
+func getCDIDeviceInfo(resp *pluginapi.ContainerAllocateResponse, knownCDIDevices sets.Set[string]) []kubecontainer.CDIDevice {
+	var cdiDevices []kubecontainer.CDIDevice
+	for _, cdiDevice := range resp.CDIDevices {
+		if knownCDIDevices.Has(cdiDevice.Name) {
+			klog.V(4).InfoS("Skip existing CDI Device", "name", cdiDevice.Name)
+			continue
+		}
+		klog.V(4).InfoS("Add CDI device", "name", cdiDevice.Name)
+		knownCDIDevices.Insert(cdiDevice.Name)
+
+		device := kubecontainer.CDIDevice{
+			Name: cdiDevice.Name,
+		}
+		cdiDevices = append(cdiDevices, device)
+	}
+
+	return cdiDevices
+}
+
 // getContainerDevices returns the devices assigned to the provided container for all ResourceNames
 func (pdev *podDevices) getContainerDevices(podUID, contName string) ResourceDeviceInstances {
 	pdev.RLock()

pkg/kubelet/cm/devicemanager/pod_devices_test.go

@@ -20,11 +20,16 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 )
 
 func TestGetContainerDevices(t *testing.T) {
@@ -153,3 +158,137 @@ func expectResourceDeviceInstances(t *testing.T, resp ResourceDeviceInstances, e
 		t.Errorf("expected %q got %q", expected, got)
 	}
 }
+
+func TestDeviceRunContainerOptions(t *testing.T) {
+	const (
+		podUID        = "pod"
+		containerName = "container"
+		resource1     = "example1.com/resource1"
+		resource2     = "example2.com/resource2"
+	)
+	testCases := []struct {
+		description          string
+		gate                 bool
+		responsesPerResource map[string]*pluginapi.ContainerAllocateResponse
+		expected             *DeviceRunContainerOptions
+	}{
+		{
+			description: "empty response",
+			gate:        false,
+			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
+				resource1: newContainerAllocateResponse(),
+			},
+			expected: &DeviceRunContainerOptions{},
+		},
+		{
+			description: "cdi devices are ingored when feature gate is disabled",
+			gate:        false,
+			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
+				resource1: newContainerAllocateResponse(
+					withDevices(map[string]string{"/dev/r1": "/dev/r1"}),
+					withMounts(map[string]string{"/home/lib1": "/home/lib1"}),
+					withEnvs(map[string]string{"ENV1": "VALUE1"}),
+					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
+				),
+			},
+			expected: &DeviceRunContainerOptions{
+				Devices: []kubecontainer.DeviceInfo{
+					{PathOnHost: "/dev/r1", PathInContainer: "/dev/r1", Permissions: "mrw"},
+				},
+				Mounts: []kubecontainer.Mount{
+					{Name: "/home/lib1", HostPath: "/home/lib1", ContainerPath: "/home/lib1", ReadOnly: true},
+				},
+				Envs: []kubecontainer.EnvVar{
+					{Name: "ENV1", Value: "VALUE1"},
+				},
+			},
+		},
+		{
+			description: "cdi devices are handled when feature gate is enabled",
+			gate:        true,
+			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
+				resource1: newContainerAllocateResponse(
+					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
+				),
+			},
+			expected: &DeviceRunContainerOptions{
+				Annotations: []kubecontainer.Annotation{
+					{Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2"},
+				},
+				CDIDevices: []kubecontainer.CDIDevice{
+					{Name: "vendor1.com/class1=device1"},
+					{Name: "vendor2.com/class2=device2"},
+				},
+			},
+		},
+		{
+			description: "cdi devices from multiple resources are handled when feature gate is enabled",
+			gate:        true,
+			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
+				resource1: newContainerAllocateResponse(
+					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
+				),
+				resource2: newContainerAllocateResponse(
+					withCDIDevices("vendor3.com/class3=device3", "vendor4.com/class4=device4"),
+				),
+			},
+			expected: &DeviceRunContainerOptions{
+				Annotations: []kubecontainer.Annotation{
+					{Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3,vendor4.com/class4=device4"},
+				},
+				CDIDevices: []kubecontainer.CDIDevice{
+					{Name: "vendor1.com/class1=device1"},
+					{Name: "vendor2.com/class2=device2"},
+					{Name: "vendor3.com/class3=device3"},
+					{Name: "vendor4.com/class4=device4"},
+				},
+			},
+		},
+		{
+			description: "duplicate cdi devices are skipped",
+			gate:        true,
+			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
+				resource1: newContainerAllocateResponse(
+					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
+				),
+				resource2: newContainerAllocateResponse(
+					withCDIDevices("vendor2.com/class2=device2", "vendor3.com/class3=device3"),
+				),
+			},
+			expected: &DeviceRunContainerOptions{
+				Annotations: []kubecontainer.Annotation{
+					{Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3"},
+				},
+				CDIDevices: []kubecontainer.CDIDevice{
+					{Name: "vendor1.com/class1=device1"},
+					{Name: "vendor2.com/class2=device2"},
+					{Name: "vendor3.com/class3=device3"},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			as := assert.New(t)
+
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DevicePluginCDIDevices, tc.gate)()
+			podDevices := newPodDevices()
+			for resourceName, response := range tc.responsesPerResource {
+				podDevices.insert("pod", "container", resourceName,
+					nil,
+					response,
+				)
+			}
+			opts := podDevices.deviceRunContainerOptions(podUID, containerName)
+
+			// The exact ordering of the options depends on the order of the resources in the map.
+			// We therefore use `ElementsMatch` instead of `Equal` on the member slices.
+			as.ElementsMatch(tc.expected.Annotations, opts.Annotations)
+			as.ElementsMatch(tc.expected.CDIDevices, opts.CDIDevices)
+			as.ElementsMatch(tc.expected.Devices, opts.Devices)
+			as.ElementsMatch(tc.expected.Envs, opts.Envs)
+			as.ElementsMatch(tc.expected.Mounts, opts.Mounts)
+		})
+	}
+}

pkg/kubelet/cm/devicemanager/types.go

@@ -91,6 +91,8 @@ type DeviceRunContainerOptions struct {
 	Devices []kubecontainer.DeviceInfo
 	// The Annotations for the container
 	Annotations []kubecontainer.Annotation
+	// CDI Devices for the container
+	CDIDevices []kubecontainer.CDIDevice
 }
 
 // TODO: evaluate whether we need this error definition.

staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.pb.go

@@ -814,6 +814,56 @@ func (m *ContainerAllocateRequest) GetDevicesIDs() []string {
 	return nil
 }
 
+// CDIDevice specifies a CDI device information.
+type CDIDevice struct {
+	// Fully qualified CDI device name
+	// for example: vendor.com/gpu=gpudevice1
+	// see more details in the CDI specification:
+	// https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md
+	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *CDIDevice) Reset()      { *m = CDIDevice{} }
+func (*CDIDevice) ProtoMessage() {}
+func (*CDIDevice) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{15}
+}
+func (m *CDIDevice) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *CDIDevice) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_CDIDevice.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *CDIDevice) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_CDIDevice.Merge(m, src)
+}
+func (m *CDIDevice) XXX_Size() int {
+	return m.Size()
+}
+func (m *CDIDevice) XXX_DiscardUnknown() {
+	xxx_messageInfo_CDIDevice.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CDIDevice proto.InternalMessageInfo
+
+func (m *CDIDevice) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
 // AllocateResponse includes the artifacts that needs to be injected into
 // a container for accessing 'deviceIDs' that were mentioned as part of
 // 'AllocateRequest'.
@@ -831,7 +881,7 @@ type AllocateResponse struct {
 func (m *AllocateResponse) Reset()      { *m = AllocateResponse{} }
 func (*AllocateResponse) ProtoMessage() {}
 func (*AllocateResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_00212fb1f9d3bf1c, []int{15}
+	return fileDescriptor_00212fb1f9d3bf1c, []int{16}
 }
 func (m *AllocateResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -876,6 +926,8 @@ type ContainerAllocateResponse struct {
 	Devices []*DeviceSpec `protobuf:"bytes,3,rep,name=devices,proto3" json:"devices,omitempty"`
 	// Container annotations to pass to the container runtime
 	Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// CDI devices for the container.
+	CDIDevices           []*CDIDevice `protobuf:"bytes,5,rep,name=cdi_devices,json=cdiDevices,proto3" json:"cdi_devices,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
 	XXX_sizecache        int32        `json:"-"`
 }
@@ -883,7 +935,7 @@ type ContainerAllocateResponse struct {
 func (m *ContainerAllocateResponse) Reset()      { *m = ContainerAllocateResponse{} }
 func (*ContainerAllocateResponse) ProtoMessage() {}
 func (*ContainerAllocateResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_00212fb1f9d3bf1c, []int{16}
+	return fileDescriptor_00212fb1f9d3bf1c, []int{17}
 }
 func (m *ContainerAllocateResponse) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -940,6 +992,13 @@ func (m *ContainerAllocateResponse) GetAnnotations() map[string]string {
 	return nil
 }
 
+func (m *ContainerAllocateResponse) GetCDIDevices() []*CDIDevice {
+	if m != nil {
+		return m.CDIDevices
+	}
+	return nil
+}
+
 // Mount specifies a host volume to mount into a container.
 // where device library or tools are installed on host and container
 type Mount struct {
@@ -956,7 +1015,7 @@ type Mount struct {
 func (m *Mount) Reset()      { *m = Mount{} }
 func (*Mount) ProtoMessage() {}
 func (*Mount) Descriptor() ([]byte, []int) {
-	return fileDescriptor_00212fb1f9d3bf1c, []int{17}
+	return fileDescriptor_00212fb1f9d3bf1c, []int{18}
 }
 func (m *Mount) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1024,7 +1083,7 @@ type DeviceSpec struct {
 func (m *DeviceSpec) Reset()      { *m = DeviceSpec{} }
 func (*DeviceSpec) ProtoMessage() {}
 func (*DeviceSpec) Descriptor() ([]byte, []int) {
-	return fileDescriptor_00212fb1f9d3bf1c, []int{18}
+	return fileDescriptor_00212fb1f9d3bf1c, []int{19}
 }
 func (m *DeviceSpec) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -1090,6 +1149,7 @@ func init() {
 	proto.RegisterType((*ContainerPreferredAllocationResponse)(nil), "v1beta1.ContainerPreferredAllocationResponse")
 	proto.RegisterType((*AllocateRequest)(nil), "v1beta1.AllocateRequest")
 	proto.RegisterType((*ContainerAllocateRequest)(nil), "v1beta1.ContainerAllocateRequest")
+	proto.RegisterType((*CDIDevice)(nil), "v1beta1.CDIDevice")
 	proto.RegisterType((*AllocateResponse)(nil), "v1beta1.AllocateResponse")
 	proto.RegisterType((*ContainerAllocateResponse)(nil), "v1beta1.ContainerAllocateResponse")
 	proto.RegisterMapType((map[string]string)(nil), "v1beta1.ContainerAllocateResponse.AnnotationsEntry")
@@ -1101,73 +1161,75 @@ func init() {
 func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) }
 
 var fileDescriptor_00212fb1f9d3bf1c = []byte{
-	// 1044 bytes of a gzipped FileDescriptorProto
+	// 1088 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xef, 0x6e, 0x1b, 0x45,
-	0x14, 0xce, 0xc6, 0x4d, 0x62, 0x1f, 0xa7, 0xf9, 0x99, 0x84, 0xc8, 0xd9, 0x14, 0x37, 0x9d, 0x14,
+	0x10, 0xcf, 0xc5, 0x71, 0x62, 0x8f, 0xd3, 0xfc, 0xd9, 0x84, 0xc8, 0x71, 0x8a, 0x93, 0x6e, 0x0a,
-	0x1a, 0xa4, 0xc4, 0x26, 0x2e, 0x6a, 0x2b, 0x2e, 0x10, 0x2e, 0x0e, 0x60, 0x42, 0xd3, 0x68, 0x43,
+	0x0d, 0x52, 0x62, 0x13, 0x17, 0xb5, 0x15, 0x1f, 0x10, 0x6e, 0x1c, 0xc0, 0x84, 0xa6, 0xd1, 0x85,
-	0x85, 0x04, 0x08, 0x6b, 0xbc, 0x3b, 0xb1, 0x57, 0x59, 0xcf, 0x2c, 0x3b, 0x63, 0x4b, 0xae, 0x84,
+	0x0a, 0x09, 0x10, 0xa7, 0xf3, 0xdd, 0xc6, 0x3e, 0xe5, 0xbc, 0x7b, 0xdc, 0xae, 0x2d, 0xb9, 0x12,
-	0xc4, 0x05, 0x0f, 0xd0, 0x77, 0x80, 0x57, 0xe0, 0x1d, 0x7a, 0xc9, 0x25, 0x57, 0x88, 0x9a, 0x17,
+	0x12, 0x8f, 0xd0, 0x77, 0x80, 0x57, 0xe0, 0x1d, 0xca, 0x37, 0x3e, 0xf2, 0x09, 0xd1, 0xf0, 0x22,
-	0x41, 0x9e, 0xd9, 0x3f, 0x6d, 0x36, 0x56, 0x2a, 0x71, 0xe7, 0x39, 0xe7, 0x7c, 0xe7, 0xe7, 0x3b,
+	0xc8, 0xbb, 0xb7, 0x77, 0xa7, 0x8b, 0x63, 0xa5, 0x52, 0xbf, 0xdd, 0xce, 0xcc, 0x6f, 0x66, 0xf6,
-	0x67, 0xcf, 0x31, 0x94, 0x88, 0xef, 0xd6, 0xfc, 0x80, 0x4b, 0x8e, 0x96, 0x46, 0x47, 0x5d, 0x2a,
+	0x37, 0xb3, 0x33, 0x07, 0x45, 0x3b, 0xf0, 0x6a, 0x41, 0xc8, 0x04, 0x43, 0x0b, 0xc3, 0xc3, 0x0e,
-	0xc9, 0x91, 0x79, 0xd8, 0x73, 0x65, 0x7f, 0xd8, 0xad, 0xd9, 0x7c, 0x50, 0xef, 0xf1, 0x1e, 0xaf,
+	0x11, 0xf6, 0x61, 0xe5, 0xa0, 0xeb, 0x89, 0xde, 0xa0, 0x53, 0x73, 0x58, 0xbf, 0xde, 0x65, 0x5d,
-	0x2b, 0x7d, 0x77, 0x78, 0xa1, 0x5e, 0xea, 0xa1, 0x7e, 0x69, 0x1c, 0x7e, 0x65, 0xc0, 0x46, 0x8b,
+	0x56, 0x97, 0xfa, 0xce, 0xe0, 0x42, 0x9e, 0xe4, 0x41, 0x7e, 0x29, 0x1c, 0x7e, 0x65, 0xc0, 0x5a,
-	0x8e, 0x5c, 0x9b, 0x9e, 0x79, 0xc3, 0x9e, 0xcb, 0x9e, 0xfb, 0xd2, 0xe5, 0x4c, 0xa0, 0x03, 0x40,
+	0x8b, 0x0c, 0x3d, 0x87, 0x9c, 0xf9, 0x83, 0xae, 0x47, 0x9f, 0x07, 0xc2, 0x63, 0x94, 0xa3, 0x7d,
-	0x7e, 0x40, 0x3b, 0x42, 0x92, 0x40, 0x76, 0x02, 0xfa, 0xd3, 0xd0, 0x0d, 0xa8, 0x53, 0x31, 0x76,
+	0x40, 0x41, 0x48, 0x2c, 0x2e, 0xec, 0x50, 0x58, 0x21, 0xf9, 0x79, 0xe0, 0x85, 0xc4, 0x2d, 0x1b,
-	0x8d, 0xfd, 0xa2, 0xb5, 0xe6, 0x07, 0xf4, 0x7c, 0xaa, 0xb0, 0x42, 0x39, 0x3a, 0x01, 0xdc, 0xa3,
+	0x3b, 0xc6, 0x5e, 0xc1, 0x5c, 0x09, 0x42, 0x72, 0x3e, 0x56, 0x98, 0x91, 0x1c, 0x9d, 0x00, 0xee,
-	0xb2, 0xe3, 0x07, 0xf4, 0x82, 0x06, 0x01, 0x75, 0x3a, 0xc4, 0xf3, 0xb8, 0x4d, 0xa6, 0xae, 0x3a,
+	0x12, 0x61, 0x05, 0x21, 0xb9, 0x20, 0x61, 0x48, 0x5c, 0xcb, 0xf6, 0x7d, 0xe6, 0xd8, 0x63, 0x57,
-	0x64, 0x44, 0x5c, 0x8f, 0x74, 0x3d, 0x5a, 0x99, 0x57, 0xe8, 0xbb, 0x3d, 0x2a, 0xcf, 0x22, 0xc3,
+	0x96, 0x3d, 0xb4, 0x3d, 0xdf, 0xee, 0xf8, 0xa4, 0x3c, 0x2b, 0xd1, 0xdb, 0x5d, 0x22, 0xce, 0xb4,
-	0x66, 0x6c, 0xd7, 0x8c, 0xcc, 0xf0, 0xef, 0x06, 0xac, 0x5a, 0xb4, 0xe7, 0x0a, 0x49, 0x83, 0x69,
+	0x61, 0x33, 0xb6, 0x6b, 0x6a, 0x33, 0xfc, 0xbb, 0x01, 0xcb, 0x26, 0xe9, 0x7a, 0x5c, 0x90, 0x70,
-	0x04, 0x2a, 0x24, 0xaa, 0xc0, 0xd2, 0x88, 0x06, 0xc2, 0xe5, 0x4c, 0xe5, 0x50, 0xb2, 0xa2, 0x27,
+	0x1c, 0x81, 0x70, 0x81, 0xca, 0xb0, 0x30, 0x24, 0x21, 0xf7, 0x18, 0x95, 0x39, 0x14, 0x4d, 0x7d,
-	0x32, 0xa1, 0x48, 0x99, 0xe3, 0x73, 0x97, 0x49, 0x15, 0xa0, 0x64, 0xc5, 0x6f, 0xb4, 0x07, 0xb7,
+	0x44, 0x15, 0x28, 0x10, 0xea, 0x06, 0xcc, 0xa3, 0x42, 0x06, 0x28, 0x9a, 0xf1, 0x19, 0xed, 0xc2,
-	0x03, 0x2a, 0xf8, 0x30, 0xb0, 0x69, 0x87, 0x91, 0x01, 0xad, 0x14, 0x94, 0xc1, 0x72, 0x24, 0x3c,
+	0x9d, 0x90, 0x70, 0x36, 0x08, 0x1d, 0x62, 0x51, 0xbb, 0x4f, 0xca, 0x39, 0x69, 0xb0, 0xa8, 0x85,
-	0x25, 0x03, 0x8a, 0x1e, 0xc1, 0x12, 0xd7, 0x45, 0x57, 0x6e, 0xed, 0x1a, 0xfb, 0xe5, 0xc6, 0x9d,
+	0xa7, 0x76, 0x9f, 0xa0, 0x47, 0xb0, 0xc0, 0xd4, 0xa5, 0xcb, 0x73, 0x3b, 0xc6, 0x5e, 0xa9, 0x71,
-	0x5a, 0xc8, 0x65, 0x2d, 0x87, 0x18, 0x2b, 0x32, 0xc6, 0x4b, 0xb0, 0x70, 0x3c, 0xf0, 0xe5, 0x18,
+	0xb7, 0x16, 0x71, 0x59, 0x9b, 0x40, 0x8c, 0xa9, 0x8d, 0xf1, 0x02, 0xe4, 0x8f, 0xfb, 0x81, 0x18,
-	0x37, 0x61, 0xf3, 0x6b, 0x57, 0xc8, 0x26, 0x73, 0xbe, 0x25, 0xd2, 0xee, 0x5b, 0x54, 0xf8, 0x9c,
+	0xe1, 0x26, 0xac, 0x7f, 0xe3, 0x71, 0xd1, 0xa4, 0xee, 0x77, 0xb6, 0x70, 0x7a, 0x26, 0xe1, 0x01,
-	0x09, 0x8a, 0x3e, 0x80, 0x25, 0x47, 0x39, 0x10, 0x15, 0x63, 0xb7, 0xb0, 0x5f, 0x6e, 0xac, 0x66,
+	0xa3, 0x9c, 0xa0, 0x8f, 0x60, 0xc1, 0x95, 0x0e, 0x78, 0xd9, 0xd8, 0xc9, 0xed, 0x95, 0x1a, 0xcb,
-	0x1c, 0x5b, 0x91, 0x1e, 0x3f, 0x86, 0xe5, 0x6f, 0xb8, 0xcf, 0x3d, 0xde, 0x1b, 0xb7, 0xd9, 0x05,
+	0x19, 0xc7, 0xa6, 0xd6, 0xe3, 0xc7, 0xb0, 0xf8, 0x2d, 0x0b, 0x98, 0xcf, 0xba, 0xa3, 0x36, 0xbd,
-	0x47, 0x0f, 0x60, 0x81, 0x71, 0x27, 0x06, 0xae, 0xc7, 0xc0, 0xd3, 0x17, 0xcf, 0x9a, 0xa7, 0xdc,
+	0x60, 0xe8, 0x01, 0xe4, 0x29, 0x73, 0x63, 0xe0, 0x6a, 0x0c, 0x3c, 0x7d, 0xf1, 0xac, 0x79, 0xca,
-	0xa1, 0x96, 0xd6, 0x63, 0x13, 0x8a, 0x91, 0x08, 0xad, 0xc0, 0x7c, 0xbb, 0xa5, 0xe8, 0x29, 0x58,
+	0x5c, 0x62, 0x2a, 0x3d, 0xae, 0x40, 0x41, 0x8b, 0xd0, 0x12, 0xcc, 0xb6, 0x5b, 0x92, 0x9e, 0x9c,
-	0xf3, 0xed, 0x16, 0xb6, 0x61, 0x51, 0xc7, 0x49, 0x69, 0x4a, 0x53, 0x0d, 0xda, 0x82, 0xc5, 0x3e,
+	0x39, 0xdb, 0x6e, 0x61, 0x07, 0xe6, 0x55, 0x9c, 0x94, 0xa6, 0x38, 0xd6, 0xa0, 0x0d, 0x98, 0xef,
-	0x25, 0x9e, 0xec, 0x87, 0x8c, 0x85, 0x2f, 0x74, 0x04, 0x45, 0x19, 0xa6, 0xa1, 0xa8, 0x2a, 0x37,
+	0x11, 0xdb, 0x17, 0xbd, 0x88, 0xb1, 0xe8, 0x84, 0x0e, 0xa1, 0x20, 0xa2, 0x34, 0x24, 0x55, 0xa5,
-	0xde, 0x89, 0x23, 0xa7, 0xf3, 0xb3, 0x62, 0x33, 0x7c, 0x02, 0x95, 0xb3, 0x70, 0x1a, 0x3e, 0xe3,
+	0xc6, 0x7b, 0x71, 0xe4, 0x74, 0x7e, 0x66, 0x6c, 0x86, 0x4f, 0xa0, 0x7c, 0x16, 0x75, 0xc3, 0x11,
-	0x4c, 0x12, 0x97, 0x25, 0x4d, 0xab, 0x43, 0x39, 0x2c, 0xb0, 0xe3, 0x3a, 0xba, 0x96, 0xd2, 0xd3,
+	0xa3, 0xc2, 0xf6, 0x68, 0x52, 0xb4, 0x3a, 0x94, 0xa2, 0x0b, 0x5a, 0x9e, 0xab, 0xee, 0x52, 0x7c,
-	0x95, 0xc9, 0xdf, 0x77, 0x41, 0xe7, 0x25, 0xda, 0x2d, 0x61, 0x41, 0x68, 0xd2, 0x76, 0x04, 0xde,
+	0xba, 0x74, 0xf5, 0xcf, 0x36, 0xa8, 0xbc, 0x78, 0xbb, 0xc5, 0x4d, 0x88, 0x4c, 0xda, 0x2e, 0xc7,
-	0x81, 0xed, 0x1c, 0x67, 0x9a, 0x4e, 0x3c, 0x06, 0x33, 0x67, 0x6c, 0xa2, 0x58, 0xdf, 0x03, 0xb2,
+	0x5b, 0xb0, 0x39, 0xc1, 0x99, 0xa2, 0x13, 0x8f, 0xa0, 0x32, 0xa1, 0x6d, 0x74, 0xac, 0x1f, 0x00,
-	0x23, 0x88, 0x9a, 0x57, 0x2a, 0x64, 0x44, 0xdf, 0x41, 0x5c, 0x44, 0xec, 0xf5, 0x7a, 0x4f, 0xd6,
+	0x39, 0x1a, 0x22, 0xfb, 0x95, 0x70, 0xa1, 0xe9, 0xdb, 0x8f, 0x2f, 0x11, 0x7b, 0xbd, 0xd9, 0x93,
-	0xba, 0x9d, 0xa9, 0x43, 0xe0, 0x3f, 0x0c, 0xd8, 0xbb, 0x01, 0x14, 0xd5, 0x61, 0x23, 0x9e, 0xf6,
+	0xb9, 0xea, 0x64, 0xee, 0xc1, 0xf1, 0x1f, 0x06, 0xec, 0xde, 0x02, 0x8a, 0xea, 0xb0, 0x16, 0x77,
-	0x8e, 0xae, 0xab, 0xdd, 0x0a, 0x0b, 0xb7, 0x50, 0xac, 0x6a, 0x45, 0x1a, 0xf4, 0x11, 0x6c, 0x0d,
+	0xbb, 0xa5, 0xee, 0xd5, 0x6e, 0x45, 0x17, 0x37, 0x51, 0xac, 0x6a, 0x69, 0x0d, 0xfa, 0x04, 0x36,
-	0x86, 0x42, 0x76, 0x5c, 0x66, 0x7b, 0x43, 0x27, 0x8d, 0x99, 0x57, 0x98, 0xcd, 0xa9, 0xb6, 0xad,
+	0xfa, 0x03, 0x2e, 0x2c, 0x8f, 0x3a, 0xfe, 0xc0, 0x4d, 0x63, 0x66, 0x25, 0x66, 0x7d, 0xac, 0x6d,
-	0x95, 0x09, 0xea, 0x01, 0xac, 0xa6, 0xbe, 0x2f, 0xe1, 0xbe, 0xd4, 0x83, 0xbd, 0x60, 0xad, 0x24,
+	0x2b, 0x65, 0x82, 0x7a, 0x00, 0xcb, 0xa9, 0xf7, 0xc5, 0xbd, 0x97, 0xaa, 0xb1, 0xf3, 0xe6, 0x52,
-	0xe2, 0x73, 0xf7, 0x25, 0xc5, 0x3f, 0xc3, 0x4e, 0x6e, 0xb6, 0xe1, 0x80, 0xfe, 0x08, 0x1b, 0x69,
+	0x22, 0x3e, 0xf7, 0x5e, 0x12, 0xfc, 0x0b, 0x6c, 0x4d, 0xcc, 0x36, 0x6a, 0xd0, 0x9f, 0x60, 0x2d,
-	0xce, 0xb4, 0x34, 0x22, 0xed, 0xf0, 0x86, 0xa4, 0x69, 0x94, 0x85, 0xec, 0x6c, 0xc3, 0x04, 0x6e,
+	0xcd, 0x99, 0x92, 0x6a, 0xd2, 0x0e, 0x6e, 0x49, 0x9a, 0x42, 0x99, 0xc8, 0xc9, 0x16, 0x8c, 0xe3,
-	0xc1, 0xfd, 0x9b, 0x60, 0xd1, 0x1d, 0x28, 0x65, 0xc9, 0x4a, 0x04, 0xd8, 0x86, 0xd5, 0x10, 0x43,
+	0x16, 0xdc, 0xbf, 0x0d, 0x16, 0xdd, 0x85, 0x62, 0x96, 0xac, 0x44, 0x80, 0x1d, 0x58, 0x8e, 0x30,
-	0x23, 0x9e, 0xcf, 0x66, 0x34, 0xfb, 0xde, 0xd5, 0xbc, 0x33, 0xf0, 0xbc, 0x0e, 0x9f, 0x40, 0xe5,
+	0x44, 0xf3, 0x7c, 0x36, 0xa5, 0xd8, 0xf7, 0xae, 0xe7, 0x9d, 0x81, 0x4f, 0xaa, 0xf0, 0x09, 0x94,
-	0x3a, 0xf3, 0xb7, 0x1f, 0xe3, 0x1e, 0xac, 0x25, 0x3e, 0xc2, 0x1a, 0xcf, 0x67, 0x71, 0x8d, 0x67,
+	0x6f, 0x32, 0x7f, 0xfb, 0x36, 0xde, 0x86, 0xe2, 0x51, 0xab, 0x1d, 0xbd, 0x3d, 0x04, 0x73, 0x72,
-	0xe5, 0x3c, 0x83, 0xe0, 0x5f, 0x0b, 0xb0, 0x7d, 0x2d, 0x02, 0x7d, 0x0a, 0xb7, 0x28, 0x1b, 0xcd,
+	0xf4, 0xa8, 0xd7, 0x27, 0xbf, 0x71, 0x17, 0x56, 0x92, 0x20, 0x11, 0x09, 0xe7, 0xd3, 0x8a, 0x81,
-	0xf8, 0x08, 0xb2, 0x88, 0xda, 0x31, 0x1b, 0x89, 0x63, 0x26, 0x83, 0xb1, 0xa5, 0x90, 0xe8, 0x7d,
+	0xa7, 0x5d, 0x6a, 0x4a, 0x05, 0xfe, 0xcc, 0xc1, 0xe6, 0x8d, 0x08, 0xf4, 0x39, 0xcc, 0x11, 0x3a,
-	0x58, 0x1c, 0xf0, 0x21, 0x93, 0x7a, 0x1c, 0xcb, 0x8d, 0x95, 0xd8, 0xc7, 0xb3, 0xa9, 0xd8, 0x0a,
+	0x9c, 0xf2, 0x4a, 0xb2, 0x88, 0xda, 0x31, 0x1d, 0xf2, 0x63, 0x2a, 0xc2, 0x91, 0x29, 0x91, 0xe8,
-	0xb5, 0xe8, 0x30, 0xd9, 0x74, 0x05, 0x65, 0xb8, 0x91, 0xd9, 0x74, 0xe7, 0x3e, 0xb5, 0xe3, 0x6d,
+	0x43, 0x98, 0xef, 0xb3, 0x01, 0x15, 0xaa, 0x5f, 0x4b, 0x8d, 0xa5, 0xd8, 0xc7, 0xb3, 0xb1, 0xd8,
-	0x87, 0x5e, 0x40, 0x99, 0x30, 0xc6, 0x25, 0x89, 0xb6, 0xee, 0x14, 0xf2, 0xf0, 0x06, 0xf9, 0x35,
+	0x8c, 0xb4, 0xe8, 0x20, 0x19, 0x85, 0x39, 0x69, 0xb8, 0x96, 0x19, 0x85, 0xe7, 0x01, 0x71, 0xe2,
-	0x13, 0x94, 0x4e, 0x33, 0xed, 0xc7, 0x7c, 0x0c, 0xa5, 0xb8, 0x00, 0xb4, 0x06, 0x85, 0x4b, 0x3a,
+	0x71, 0x88, 0x5e, 0x40, 0xc9, 0xa6, 0x94, 0x09, 0x5b, 0x8f, 0xe5, 0x31, 0xe4, 0xe1, 0x2d, 0xf2,
-	0x0e, 0x77, 0xde, 0xf4, 0x27, 0xda, 0x84, 0x85, 0x11, 0xf1, 0x86, 0x34, 0xdc, 0x79, 0xfa, 0xf1,
+	0x6b, 0x26, 0x28, 0x95, 0x66, 0xda, 0x0f, 0x3a, 0x82, 0x92, 0xe3, 0x7a, 0x96, 0xce, 0x24, 0x2f,
-	0xf1, 0xfc, 0x13, 0xc3, 0xfc, 0x04, 0xd6, 0xb2, 0x9e, 0xdf, 0x06, 0x8f, 0xfb, 0xb0, 0xa0, 0xf8,
+	0xdd, 0xa2, 0xc4, 0xad, 0xae, 0x99, 0x2a, 0x6e, 0x7c, 0xe4, 0x26, 0x38, 0xae, 0x17, 0x7d, 0x57,
-	0x40, 0xef, 0xc1, 0x4a, 0xd2, 0x64, 0x9f, 0xc8, 0x7e, 0x88, 0xbf, 0x1d, 0x4b, 0xcf, 0x88, 0xec,
+	0x1e, 0x43, 0x31, 0x66, 0x01, 0xad, 0x40, 0xee, 0x92, 0x8c, 0xa2, 0xda, 0x8e, 0x3f, 0xd1, 0x3a,
-	0xa3, 0x1d, 0x28, 0xf5, 0xb9, 0x90, 0xda, 0x22, 0xbc, 0x59, 0x53, 0x41, 0xa4, 0x0c, 0x28, 0x71,
+	0xe4, 0x87, 0xb6, 0x3f, 0x20, 0xd1, 0x64, 0x55, 0x87, 0x4f, 0x67, 0x9f, 0x18, 0x95, 0xcf, 0x60,
-	0x3a, 0x9c, 0x79, 0x7a, 0x09, 0x17, 0xad, 0xe2, 0x54, 0xf0, 0x9c, 0x79, 0x63, 0x1c, 0x00, 0x24,
+	0x25, 0x9b, 0xde, 0xdb, 0xe0, 0x71, 0x0f, 0xf2, 0x92, 0x54, 0xf4, 0x01, 0x2c, 0x25, 0x9d, 0x12,
-	0x84, 0xfe, 0x2f, 0xe1, 0x76, 0xa1, 0xec, 0xd3, 0x60, 0xe0, 0x0a, 0xa1, 0x7a, 0xa1, 0x0f, 0x64,
+	0xd8, 0xa2, 0x17, 0xe1, 0xef, 0xc4, 0xd2, 0x33, 0x5b, 0xf4, 0xd0, 0x16, 0x14, 0x7b, 0x8c, 0x0b,
-	0x5a, 0xd4, 0xf8, 0x1c, 0x96, 0xf5, 0x35, 0x0e, 0x14, 0x3f, 0xe8, 0x11, 0x14, 0xa3, 0xeb, 0x8c,
+	0x65, 0x11, 0x6d, 0xc6, 0xb1, 0x40, 0x2b, 0x43, 0x62, 0xbb, 0x16, 0xa3, 0xbe, 0x1a, 0xf5, 0x05,
-	0x2a, 0x71, 0xd3, 0x32, 0x07, 0xdb, 0x4c, 0x46, 0x45, 0x1f, 0xc9, 0xb9, 0xc6, 0x6f, 0x05, 0x58,
+	0xb3, 0x30, 0x16, 0x3c, 0xa7, 0xfe, 0x08, 0x87, 0x00, 0x49, 0x55, 0xde, 0x49, 0xb8, 0x1d, 0x28,
-	0x4e, 0x1f, 0x54, 0xf4, 0x25, 0x6c, 0x7d, 0x41, 0x65, 0xde, 0x9f, 0x8f, 0x0c, 0xd8, 0x9c, 0x79,
+	0x05, 0x24, 0xec, 0x7b, 0x9c, 0xcb, 0x82, 0xaa, 0x35, 0x9c, 0x16, 0x35, 0xbe, 0x80, 0x45, 0xb5,
-	0x91, 0xf1, 0x1c, 0x6a, 0xc2, 0x72, 0xfa, 0x02, 0x5f, 0xc1, 0xbf, 0x1b, 0xbf, 0xf3, 0x0e, 0x35,
+	0xf3, 0x43, 0xc9, 0x0f, 0x7a, 0x04, 0x05, 0xfd, 0x0f, 0x80, 0xca, 0x71, 0x89, 0x32, 0xbf, 0x05,
-	0x9e, 0xfb, 0xd0, 0x40, 0x54, 0x25, 0x93, 0xb3, 0xa5, 0xd0, 0x5e, 0x0c, 0xbe, 0x7e, 0xf3, 0x9b,
+	0x95, 0xa4, 0xdf, 0xd4, 0x2a, 0x9e, 0x69, 0xfc, 0x96, 0x83, 0xc5, 0xf4, 0xda, 0x46, 0x5f, 0xc1,
-	0xf7, 0x67, 0x1b, 0x45, 0x81, 0x50, 0x13, 0x8a, 0xd1, 0x54, 0xa7, 0xc8, 0xcb, 0x6c, 0x1c, 0x73,
+	0xc6, 0x97, 0x44, 0x4c, 0xfa, 0xc5, 0xc9, 0x80, 0x2b, 0x53, 0xf7, 0x3e, 0x9e, 0x41, 0x4d, 0x58,
-	0x3b, 0x47, 0x13, 0xbb, 0xf8, 0x01, 0xd6, 0xaf, 0x1c, 0x49, 0x74, 0x2f, 0x1d, 0x3f, 0xf7, 0x1a,
+	0x4c, 0xef, 0xf9, 0x6b, 0xf8, 0xf7, 0xe3, 0xf3, 0xa4, 0xdf, 0x01, 0x3c, 0xf3, 0xb1, 0x81, 0x88,
-	0x9b, 0x78, 0x96, 0x49, 0xe4, 0xfd, 0xe9, 0x57, 0xaf, 0xdf, 0x54, 0x8d, 0xbf, 0xde, 0x54, 0xe7,
+	0x4c, 0x66, 0xc2, 0x2c, 0x44, 0xbb, 0x31, 0xf8, 0xe6, 0xfd, 0x52, 0xb9, 0x3f, 0xdd, 0x48, 0x07,
-	0x7e, 0x99, 0x54, 0x8d, 0xd7, 0x93, 0xaa, 0xf1, 0xe7, 0xa4, 0x6a, 0xfc, 0x33, 0xa9, 0x1a, 0xaf,
+	0x42, 0x4d, 0x28, 0xe8, 0xa7, 0x91, 0x22, 0x2f, 0x33, 0xd7, 0x2a, 0x9b, 0x13, 0x34, 0xb1, 0x8b,
-	0xfe, 0xad, 0xce, 0x7d, 0x77, 0x70, 0xf9, 0x44, 0xd4, 0x5c, 0x5e, 0xbf, 0x1c, 0x76, 0xa9, 0x47,
+	0x1f, 0x61, 0xf5, 0xda, 0x2a, 0x46, 0xf7, 0xd2, 0xf1, 0x27, 0xee, 0xfc, 0x0a, 0x9e, 0x66, 0xa2,
-	0x65, 0xdd, 0xbf, 0xec, 0xd5, 0x89, 0xef, 0x8a, 0xba, 0xfe, 0xb4, 0x7d, 0xd5, 0x97, 0x7a, 0x18,
+	0xbd, 0x3f, 0xfd, 0xfa, 0xf5, 0x9b, 0xaa, 0xf1, 0xf7, 0x9b, 0xea, 0xcc, 0xaf, 0x57, 0x55, 0xe3,
-	0xa7, 0xbb, 0xa8, 0xfe, 0x62, 0x3e, 0xfc, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe2, 0xf2, 0x09, 0x79,
+	0xf5, 0x55, 0xd5, 0xf8, 0xeb, 0xaa, 0x6a, 0xfc, 0x7b, 0x55, 0x35, 0x5e, 0xfd, 0x57, 0x9d, 0xf9,
-	0xa7, 0x0a, 0x00, 0x00,
+	0x7e, 0xff, 0xf2, 0x09, 0xaf, 0x79, 0xac, 0x7e, 0x39, 0xe8, 0x10, 0x9f, 0x88, 0x7a, 0x70, 0xd9,
+	0xad, 0xdb, 0x81, 0xc7, 0xeb, 0xea, 0xe5, 0x06, 0xb2, 0x2e, 0xf5, 0x28, 0x4e, 0x67, 0x5e, 0xfe,
+	0xc8, 0x3e, 0xfc, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xe0, 0x26, 0x52, 0x96, 0x0d, 0x0b, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -2075,6 +2137,36 @@ func (m *ContainerAllocateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error
 	return len(dAtA) - i, nil
 }
 
+func (m *CDIDevice) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *CDIDevice) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *CDIDevice) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Name) > 0 {
+		i -= len(m.Name)
+		copy(dAtA[i:], m.Name)
+		i = encodeVarintApi(dAtA, i, uint64(len(m.Name)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
 func (m *AllocateResponse) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -2132,6 +2224,20 @@ func (m *ContainerAllocateResponse) MarshalToSizedBuffer(dAtA []byte) (int, erro
 	_ = i
 	var l int
 	_ = l
+	if len(m.CDIDevices) > 0 {
+		for iNdEx := len(m.CDIDevices) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.CDIDevices[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintApi(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x2a
+		}
+	}
 	if len(m.Annotations) > 0 {
 		for k := range m.Annotations {
 			v := m.Annotations[k]
@@ -2538,6 +2644,19 @@ func (m *ContainerAllocateRequest) Size() (n int) {
 	return n
 }
 
+func (m *CDIDevice) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Name)
+	if l > 0 {
+		n += 1 + l + sovApi(uint64(l))
+	}
+	return n
+}
+
 func (m *AllocateResponse) Size() (n int) {
 	if m == nil {
 		return 0
@@ -2587,6 +2706,12 @@ func (m *ContainerAllocateResponse) Size() (n int) {
 			n += mapEntrySize + 1 + sovApi(uint64(mapEntrySize))
 		}
 	}
+	if len(m.CDIDevices) > 0 {
+		for _, e := range m.CDIDevices {
+			l = e.Size()
+			n += 1 + l + sovApi(uint64(l))
+		}
+	}
 	return n
 }
 
@@ -2818,6 +2943,16 @@ func (this *ContainerAllocateRequest) String() string {
 	}, "")
 	return s
 }
+func (this *CDIDevice) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&CDIDevice{`,
+		`Name:` + fmt.Sprintf("%v", this.Name) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func (this *AllocateResponse) String() string {
 	if this == nil {
 		return "nil"
@@ -2847,6 +2982,11 @@ func (this *ContainerAllocateResponse) String() string {
 		repeatedStringForDevices += strings.Replace(f.String(), "DeviceSpec", "DeviceSpec", 1) + ","
 	}
 	repeatedStringForDevices += "}"
+	repeatedStringForCDIDevices := "[]*CDIDevice{"
+	for _, f := range this.CDIDevices {
+		repeatedStringForCDIDevices += strings.Replace(f.String(), "CDIDevice", "CDIDevice", 1) + ","
+	}
+	repeatedStringForCDIDevices += "}"
 	keysForEnvs := make([]string, 0, len(this.Envs))
 	for k := range this.Envs {
 		keysForEnvs = append(keysForEnvs, k)
@@ -2872,6 +3012,7 @@ func (this *ContainerAllocateResponse) String() string {
 		`Mounts:` + repeatedStringForMounts + `,`,
 		`Devices:` + repeatedStringForDevices + `,`,
 		`Annotations:` + mapStringForAnnotations + `,`,
+		`CDIDevices:` + repeatedStringForCDIDevices + `,`,
 		`}`,
 	}, "")
 	return s
@@ -4298,6 +4439,88 @@ func (m *ContainerAllocateRequest) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *CDIDevice) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowApi
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: CDIDevice: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: CDIDevice: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthApi
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Name = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipApi(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthApi
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *AllocateResponse) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
@@ -4733,6 +4956,40 @@ func (m *ContainerAllocateResponse) Unmarshal(dAtA []byte) error {
 			}
 			m.Annotations[mapkey] = mapvalue
 			iNdEx = postIndex
+		case 5:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field CDIDevices", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthApi
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.CDIDevices = append(m.CDIDevices, &CDIDevice{})
+			if err := m.CDIDevices[len(m.CDIDevices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipApi(dAtA[iNdEx:])

staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto

@@ -164,6 +164,15 @@ message ContainerAllocateRequest {
 	repeated string devices_ids = 1 [(gogoproto.customname) = "DevicesIDs"];
 }
 
+// CDIDevice specifies a CDI device information.
+message CDIDevice {
+	// Fully qualified CDI device name
+	// for example: vendor.com/gpu=gpudevice1
+	// see more details in the CDI specification:
+	// https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md
+	string name = 1;
+}
+
 // AllocateResponse includes the artifacts that needs to be injected into
 // a container for accessing 'deviceIDs' that were mentioned as part of
 // 'AllocateRequest'.
@@ -185,6 +194,8 @@ message ContainerAllocateResponse {
 	repeated DeviceSpec devices = 3;
 	// Container annotations to pass to the container runtime
 	map<string, string> annotations = 4;
+	// CDI devices for the container.
+	repeated CDIDevice cdi_devices = 5 [(gogoproto.customname) = "CDIDevices"];
 }
 
 // Mount specifies a host volume to mount into a container.