Merge pull request #125163 from pohly/dra-kubelet-api-version-independent-no-rest-proxy

DRA: make kubelet independent of the resource.k8s.io API version

pkg/kubelet/cm/dra/manager.go

@@ -30,7 +30,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/dynamic-resource-allocation/resourceclaim"
 	"k8s.io/klog/v2"
-	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha3"
+	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4"
 	dra "k8s.io/kubernetes/pkg/kubelet/cm/dra/plugin"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -224,13 +224,9 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
 			// Loop through all plugins and prepare for calling NodePrepareResources.
 			for _, resourceHandle := range claimInfo.ResourceHandles {
 				claim := &drapb.Claim{
-					Namespace:      claimInfo.Namespace,
+					Namespace: claimInfo.Namespace,
-					Uid:            string(claimInfo.ClaimUID),
+					Uid:       string(claimInfo.ClaimUID),
-					Name:           claimInfo.ClaimName,
+					Name:      claimInfo.ClaimName,
-					ResourceHandle: resourceHandle.Data,
-				}
-				if resourceHandle.StructuredData != nil {
-					claim.StructuredResourceHandle = []*resourceapi.StructuredResourceHandle{resourceHandle.StructuredData}
 				}
 				pluginName := resourceHandle.DriverName
 				batches[pluginName] = append(batches[pluginName], claim)
@@ -455,13 +451,9 @@ func (m *ManagerImpl) unprepareResources(podUID types.UID, namespace string, cla
 			// Loop through all plugins and prepare for calling NodeUnprepareResources.
 			for _, resourceHandle := range claimInfo.ResourceHandles {
 				claim := &drapb.Claim{
-					Namespace:      claimInfo.Namespace,
+					Namespace: claimInfo.Namespace,
-					Uid:            string(claimInfo.ClaimUID),
+					Uid:       string(claimInfo.ClaimUID),
-					Name:           claimInfo.ClaimName,
+					Name:      claimInfo.ClaimName,
-					ResourceHandle: resourceHandle.Data,
-				}
-				if resourceHandle.StructuredData != nil {
-					claim.StructuredResourceHandle = []*resourceapi.StructuredResourceHandle{resourceHandle.StructuredData}
 				}
 				pluginName := resourceHandle.DriverName
 				batches[pluginName] = append(batches[pluginName], claim)

pkg/kubelet/cm/dra/manager_test.go

@@ -36,7 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/dynamic-resource-allocation/resourceclaim"
-	drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1alpha3"
+	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4"
 	"k8s.io/kubernetes/pkg/kubelet/cm/dra/plugin"
 	"k8s.io/kubernetes/pkg/kubelet/cm/dra/state"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -48,16 +48,16 @@ const (
 )
 
 type fakeDRADriverGRPCServer struct {
-	drapbv1.UnimplementedNodeServer
+	drapb.UnimplementedNodeServer
 	driverName                 string
 	timeout                    *time.Duration
 	prepareResourceCalls       atomic.Uint32
 	unprepareResourceCalls     atomic.Uint32
-	prepareResourcesResponse   *drapbv1.NodePrepareResourcesResponse
+	prepareResourcesResponse   *drapb.NodePrepareResourcesResponse
-	unprepareResourcesResponse *drapbv1.NodeUnprepareResourcesResponse
+	unprepareResourcesResponse *drapb.NodeUnprepareResourcesResponse
 }
 
-func (s *fakeDRADriverGRPCServer) NodePrepareResources(ctx context.Context, req *drapbv1.NodePrepareResourcesRequest) (*drapbv1.NodePrepareResourcesResponse, error) {
+func (s *fakeDRADriverGRPCServer) NodePrepareResources(ctx context.Context, req *drapb.NodePrepareResourcesRequest) (*drapb.NodePrepareResourcesResponse, error) {
 	s.prepareResourceCalls.Add(1)
 
 	if s.timeout != nil {
@@ -67,8 +67,8 @@ func (s *fakeDRADriverGRPCServer) NodePrepareResources(ctx context.Context, req
 	if s.prepareResourcesResponse == nil {
 		deviceName := "claim-" + req.Claims[0].Uid
 		result := s.driverName + "/" + driverClassName + "=" + deviceName
-		return &drapbv1.NodePrepareResourcesResponse{
+		return &drapb.NodePrepareResourcesResponse{
-			Claims: map[string]*drapbv1.NodePrepareResourceResponse{
+			Claims: map[string]*drapb.NodePrepareResourceResponse{
 				req.Claims[0].Uid: {
 					CDIDevices: []string{result},
 				},
@@ -79,7 +79,7 @@ func (s *fakeDRADriverGRPCServer) NodePrepareResources(ctx context.Context, req
 	return s.prepareResourcesResponse, nil
 }
 
-func (s *fakeDRADriverGRPCServer) NodeUnprepareResources(ctx context.Context, req *drapbv1.NodeUnprepareResourcesRequest) (*drapbv1.NodeUnprepareResourcesResponse, error) {
+func (s *fakeDRADriverGRPCServer) NodeUnprepareResources(ctx context.Context, req *drapb.NodeUnprepareResourcesRequest) (*drapb.NodeUnprepareResourcesResponse, error) {
 	s.unprepareResourceCalls.Add(1)
 
 	if s.timeout != nil {
@@ -87,8 +87,8 @@ func (s *fakeDRADriverGRPCServer) NodeUnprepareResources(ctx context.Context, re
 	}
 
 	if s.unprepareResourcesResponse == nil {
-		return &drapbv1.NodeUnprepareResourcesResponse{
+		return &drapb.NodeUnprepareResourcesResponse{
-			Claims: map[string]*drapbv1.NodeUnprepareResourceResponse{
+			Claims: map[string]*drapb.NodeUnprepareResourceResponse{
 				req.Claims[0].Uid: {},
 			},
 		}, nil
@@ -108,7 +108,7 @@ type fakeDRAServerInfo struct {
 	teardownFn tearDown
 }
 
-func setupFakeDRADriverGRPCServer(shouldTimeout bool, pluginClientTimeout *time.Duration, prepareResourcesResponse *drapbv1.NodePrepareResourcesResponse, unprepareResourcesResponse *drapbv1.NodeUnprepareResourcesResponse) (fakeDRAServerInfo, error) {
+func setupFakeDRADriverGRPCServer(shouldTimeout bool, pluginClientTimeout *time.Duration, prepareResourcesResponse *drapb.NodePrepareResourcesResponse, unprepareResourcesResponse *drapb.NodeUnprepareResourcesResponse) (fakeDRAServerInfo, error) {
 	socketDir, err := os.MkdirTemp("", "dra")
 	if err != nil {
 		return fakeDRAServerInfo{
@@ -147,7 +147,7 @@ func setupFakeDRADriverGRPCServer(shouldTimeout bool, pluginClientTimeout *time.
 		fakeDRADriverGRPCServer.timeout = &timeout
 	}
 
-	drapbv1.RegisterNodeServer(s, fakeDRADriverGRPCServer)
+	drapb.RegisterNodeServer(s, fakeDRADriverGRPCServer)
 
 	go func() {
 		go s.Serve(l)
@@ -345,7 +345,7 @@ func TestPrepareResources(t *testing.T) {
 		pod                  *v1.Pod
 		claimInfo            *ClaimInfo
 		resourceClaim        *resourcev1alpha2.ResourceClaim
-		resp                 *drapbv1.NodePrepareResourcesResponse
+		resp                 *drapb.NodePrepareResourcesResponse
 		wantErr              bool
 		wantTimeout          bool
 		wantResourceSkipped  bool
@@ -484,7 +484,7 @@ func TestPrepareResources(t *testing.T) {
 					},
 				},
 			},
-			resp:                 &drapbv1.NodePrepareResourcesResponse{Claims: map[string]*drapbv1.NodePrepareResourceResponse{"test-reserved": nil}},
+			resp:                 &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"test-reserved": nil}},
 			expectedCDIDevices:   []string{},
 			ExpectedPrepareCalls: 1,
 		},
@@ -541,7 +541,7 @@ func TestPrepareResources(t *testing.T) {
 					},
 				},
 			},
-			resp:                 &drapbv1.NodePrepareResourcesResponse{Claims: map[string]*drapbv1.NodePrepareResourceResponse{"test-reserved": nil}},
+			resp:                 &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"test-reserved": nil}},
 			expectedCDIDevices:   []string{},
 			ExpectedPrepareCalls: 1,
 		},
@@ -748,8 +748,8 @@ func TestPrepareResources(t *testing.T) {
 					},
 				},
 			},
-			resp: &drapbv1.NodePrepareResourcesResponse{
+			resp: &drapb.NodePrepareResourcesResponse{
-				Claims: map[string]*drapbv1.NodePrepareResourceResponse{
+				Claims: map[string]*drapb.NodePrepareResourceResponse{
 					"test-reserved": {CDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}},
 				},
 			},
@@ -810,8 +810,8 @@ func TestPrepareResources(t *testing.T) {
 					},
 				},
 			},
-			resp: &drapbv1.NodePrepareResourcesResponse{
+			resp: &drapb.NodePrepareResourcesResponse{
-				Claims: map[string]*drapbv1.NodePrepareResourceResponse{
+				Claims: map[string]*drapb.NodePrepareResourceResponse{
 					"test-reserved": {CDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}},
 				},
 			},
@@ -884,8 +884,8 @@ func TestPrepareResources(t *testing.T) {
 					},
 				},
 			},
-			resp: &drapbv1.NodePrepareResourcesResponse{
+			resp: &drapb.NodePrepareResourcesResponse{
-				Claims: map[string]*drapbv1.NodePrepareResourceResponse{
+				Claims: map[string]*drapb.NodePrepareResourceResponse{
 					"test-reserved": {CDIDevices: []string{fmt.Sprintf("%s/%s=claim-test-reserved", driverName, driverClassName)}},
 				},
 			},
@@ -977,7 +977,7 @@ func TestUnprepareResources(t *testing.T) {
 		driverName             string
 		pod                    *v1.Pod
 		claimInfo              *ClaimInfo
-		resp                   *drapbv1.NodeUnprepareResourcesResponse
+		resp                   *drapb.NodeUnprepareResourcesResponse
 		wantErr                bool
 		wantTimeout            bool
 		wantResourceSkipped    bool
@@ -1117,7 +1117,7 @@ func TestUnprepareResources(t *testing.T) {
 					},
 				},
 			},
-			resp:                   &drapbv1.NodeUnprepareResourcesResponse{Claims: map[string]*drapbv1.NodeUnprepareResourceResponse{"test-reserved": {}}},
+			resp:                   &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"test-reserved": {}}},
 			wantErr:                true,
 			wantTimeout:            true,
 			expectedUnprepareCalls: 1,
@@ -1168,7 +1168,7 @@ func TestUnprepareResources(t *testing.T) {
 				},
 				prepared: true,
 			},
-			resp:                   &drapbv1.NodeUnprepareResourcesResponse{Claims: map[string]*drapbv1.NodeUnprepareResourceResponse{"": {}}},
+			resp:                   &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"": {}}},
 			expectedUnprepareCalls: 1,
 		},
 		{
@@ -1217,7 +1217,7 @@ func TestUnprepareResources(t *testing.T) {
 				},
 				prepared: false,
 			},
-			resp:                   &drapbv1.NodeUnprepareResourcesResponse{Claims: map[string]*drapbv1.NodeUnprepareResourceResponse{"": {}}},
+			resp:                   &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"": {}}},
 			expectedUnprepareCalls: 1,
 		},
 		{
@@ -1267,7 +1267,7 @@ func TestUnprepareResources(t *testing.T) {
 				},
 				prepared: true,
 			},
-			resp:                   &drapbv1.NodeUnprepareResourcesResponse{Claims: map[string]*drapbv1.NodeUnprepareResourceResponse{"test-reserved": nil}},
+			resp:                   &drapb.NodeUnprepareResourcesResponse{Claims: map[string]*drapb.NodeUnprepareResourceResponse{"test-reserved": nil}},
 			expectedUnprepareCalls: 1,
 		},
 	} {

pkg/kubelet/cm/dra/plugin/client.go

@@ -18,18 +18,28 @@ package plugin
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net"
+	"sync"
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
 
+	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/klog/v2"
-	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha3"
+	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4"
 )
 
 const PluginClientTimeout = 45 * time.Second
 
-func NewDRAPluginClient(pluginName string) (drapb.NodeClient, error) {
+// NewDRAPluginClient returns a wrapper around those gRPC methods of a DRA
+// driver kubelet plugin which need to be called by kubelet. The wrapper
+// handles gRPC connection management and logging. Connections are reused
+// across different NewDRAPluginClient calls.
+func NewDRAPluginClient(pluginName string) (*Plugin, error) {
 	if pluginName == "" {
 		return nil, fmt.Errorf("plugin name is empty")
 	}
@@ -42,13 +52,64 @@ func NewDRAPluginClient(pluginName string) (drapb.NodeClient, error) {
 	return existingPlugin, nil
 }
 
-func (p *plugin) NodePrepareResources(
+type Plugin struct {
+	backgroundCtx context.Context
+	cancel        func(cause error)
+
+	mutex                   sync.Mutex
+	conn                    *grpc.ClientConn
+	endpoint                string
+	highestSupportedVersion *utilversion.Version
+	clientTimeout           time.Duration
+}
+
+func (p *Plugin) getOrCreateGRPCConn() (*grpc.ClientConn, error) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+
+	if p.conn != nil {
+		return p.conn, nil
+	}
+
+	ctx := p.backgroundCtx
+	logger := klog.FromContext(ctx)
+
+	network := "unix"
+	logger.V(4).Info("Creating new gRPC connection", "protocol", network, "endpoint", p.endpoint)
+	// grpc.Dial is deprecated. grpc.NewClient should be used instead.
+	// For now this gets ignored because this function is meant to establish
+	// the connection, with the one second timeout below. Perhaps that
+	// approach should be reconsidered?
+	//nolint:staticcheck
+	conn, err := grpc.Dial(
+		p.endpoint,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) {
+			return (&net.Dialer{}).DialContext(ctx, network, target)
+		}),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	if ok := conn.WaitForStateChange(ctx, connectivity.Connecting); !ok {
+		return nil, errors.New("timed out waiting for gRPC connection to be ready")
+	}
+
+	p.conn = conn
+	return p.conn, nil
+}
+
+func (p *Plugin) NodePrepareResources(
 	ctx context.Context,
 	req *drapb.NodePrepareResourcesRequest,
 	opts ...grpc.CallOption,
 ) (*drapb.NodePrepareResourcesResponse, error) {
 	logger := klog.FromContext(ctx)
-	logger.V(4).Info(log("calling NodePrepareResources rpc"), "request", req)
+	logger.V(4).Info("Calling NodePrepareResources rpc", "request", req)
 
 	conn, err := p.getOrCreateGRPCConn()
 	if err != nil {
@@ -60,17 +121,17 @@ func (p *plugin) NodePrepareResources(
 
 	nodeClient := drapb.NewNodeClient(conn)
 	response, err := nodeClient.NodePrepareResources(ctx, req)
-	logger.V(4).Info(log("done calling NodePrepareResources rpc"), "response", response, "err", err)
+	logger.V(4).Info("Done calling NodePrepareResources rpc", "response", response, "err", err)
 	return response, err
 }
 
-func (p *plugin) NodeUnprepareResources(
+func (p *Plugin) NodeUnprepareResources(
 	ctx context.Context,
 	req *drapb.NodeUnprepareResourcesRequest,
 	opts ...grpc.CallOption,
 ) (*drapb.NodeUnprepareResourcesResponse, error) {
 	logger := klog.FromContext(ctx)
-	logger.V(4).Info(log("calling NodeUnprepareResource rpc"), "request", req)
+	logger.V(4).Info("Calling NodeUnprepareResource rpc", "request", req)
 
 	conn, err := p.getOrCreateGRPCConn()
 	if err != nil {
@@ -82,23 +143,6 @@ func (p *plugin) NodeUnprepareResources(
 
 	nodeClient := drapb.NewNodeClient(conn)
 	response, err := nodeClient.NodeUnprepareResources(ctx, req)
-	logger.V(4).Info(log("done calling NodeUnprepareResources rpc"), "response", response, "err", err)
+	logger.V(4).Info("Done calling NodeUnprepareResources rpc", "response", response, "err", err)
 	return response, err
 }
-
-func (p *plugin) NodeListAndWatchResources(
-	ctx context.Context,
-	req *drapb.NodeListAndWatchResourcesRequest,
-	opts ...grpc.CallOption,
-) (drapb.Node_NodeListAndWatchResourcesClient, error) {
-	logger := klog.FromContext(ctx)
-	logger.V(4).Info(log("calling NodeListAndWatchResources rpc"), "request", req)
-
-	conn, err := p.getOrCreateGRPCConn()
-	if err != nil {
-		return nil, err
-	}
-
-	nodeClient := drapb.NewNodeClient(conn)
-	return nodeClient.NodeListAndWatchResources(ctx, req, opts...)
-}

pkg/kubelet/cm/dra/plugin/client_test.go

@@ -27,32 +27,27 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/grpc"
-	drapbv1alpha3 "k8s.io/kubelet/pkg/apis/dra/v1alpha3"
+	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4"
+	"k8s.io/kubernetes/test/utils/ktesting"
 )
 
-type fakeV1alpha3GRPCServer struct {
+const (
-	drapbv1alpha3.UnimplementedNodeServer
+	v1alpha4Version = "v1alpha4"
+)
+
+type fakeV1alpha4GRPCServer struct {
+	drapb.UnimplementedNodeServer
 }
 
-var _ drapbv1alpha3.NodeServer = &fakeV1alpha3GRPCServer{}
+var _ drapb.NodeServer = &fakeV1alpha4GRPCServer{}
 
-func (f *fakeV1alpha3GRPCServer) NodePrepareResources(ctx context.Context, in *drapbv1alpha3.NodePrepareResourcesRequest) (*drapbv1alpha3.NodePrepareResourcesResponse, error) {
+func (f *fakeV1alpha4GRPCServer) NodePrepareResources(ctx context.Context, in *drapb.NodePrepareResourcesRequest) (*drapb.NodePrepareResourcesResponse, error) {
-	return &drapbv1alpha3.NodePrepareResourcesResponse{Claims: map[string]*drapbv1alpha3.NodePrepareResourceResponse{"dummy": {CDIDevices: []string{"dummy"}}}}, nil
+	return &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"dummy": {CDIDevices: []string{"dummy"}}}}, nil
 }
 
-func (f *fakeV1alpha3GRPCServer) NodeUnprepareResources(ctx context.Context, in *drapbv1alpha3.NodeUnprepareResourcesRequest) (*drapbv1alpha3.NodeUnprepareResourcesResponse, error) {
+func (f *fakeV1alpha4GRPCServer) NodeUnprepareResources(ctx context.Context, in *drapb.NodeUnprepareResourcesRequest) (*drapb.NodeUnprepareResourcesResponse, error) {
 
-	return &drapbv1alpha3.NodeUnprepareResourcesResponse{}, nil
+	return &drapb.NodeUnprepareResourcesResponse{}, nil
-}
-
-func (f *fakeV1alpha3GRPCServer) NodeListAndWatchResources(req *drapbv1alpha3.NodeListAndWatchResourcesRequest, srv drapbv1alpha3.Node_NodeListAndWatchResourcesServer) error {
-	if err := srv.Send(&drapbv1alpha3.NodeListAndWatchResourcesResponse{}); err != nil {
-		return err
-	}
-	if err := srv.Send(&drapbv1alpha3.NodeListAndWatchResourcesResponse{}); err != nil {
-		return err
-	}
-	return nil
 }
 
 type tearDown func()
@@ -78,9 +73,9 @@ func setupFakeGRPCServer(version string) (string, tearDown, error) {
 
 	s := grpc.NewServer()
 	switch version {
-	case v1alpha3Version:
+	case v1alpha4Version:
-		fakeGRPCServer := &fakeV1alpha3GRPCServer{}
+		fakeGRPCServer := &fakeV1alpha4GRPCServer{}
-		drapbv1alpha3.RegisterNodeServer(s, fakeGRPCServer)
+		drapb.RegisterNodeServer(s, fakeGRPCServer)
 	default:
 		return "", nil, fmt.Errorf("unsupported version: %s", version)
 	}
@@ -95,7 +90,8 @@ func setupFakeGRPCServer(version string) (string, tearDown, error) {
 }
 
 func TestGRPCConnIsReused(t *testing.T) {
-	addr, teardown, err := setupFakeGRPCServer(v1alpha3Version)
+	ctx := ktesting.Init(t)
+	addr, teardown, err := setupFakeGRPCServer(v1alpha4Version)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -105,8 +101,9 @@ func TestGRPCConnIsReused(t *testing.T) {
 	wg := sync.WaitGroup{}
 	m := sync.Mutex{}
 
-	p := &plugin{
+	p := &Plugin{
-		endpoint: addr,
+		backgroundCtx: ctx,
+		endpoint:      addr,
 	}
 
 	conn, err := p.getOrCreateGRPCConn()
@@ -135,21 +132,20 @@ func TestGRPCConnIsReused(t *testing.T) {
 				return
 			}
 
-			req := &drapbv1alpha3.NodePrepareResourcesRequest{
+			req := &drapb.NodePrepareResourcesRequest{
-				Claims: []*drapbv1alpha3.Claim{
+				Claims: []*drapb.Claim{
 					{
-						Namespace:      "dummy-namespace",
+						Namespace: "dummy-namespace",
-						Uid:            "dummy-uid",
+						Uid:       "dummy-uid",
-						Name:           "dummy-claim",
+						Name:      "dummy-claim",
-						ResourceHandle: "dummy-resource",
 					},
 				},
 			}
 			client.NodePrepareResources(context.TODO(), req)
 
-			client.(*plugin).Lock()
+			client.mutex.Lock()
-			conn := client.(*plugin).conn
+			conn := client.conn
-			client.(*plugin).Unlock()
+			client.mutex.Unlock()
 
 			m.Lock()
 			defer m.Unlock()
@@ -193,7 +189,7 @@ func TestNewDRAPluginClient(t *testing.T) {
 		{
 			description: "plugin exists",
 			setup: func(name string) tearDown {
-				draPlugins.add(name, &plugin{})
+				draPlugins.add(name, &Plugin{})
 				return func() {
 					draPlugins.delete(name)
 				}
@@ -222,23 +218,25 @@ func TestNodeUnprepareResources(t *testing.T) {
 		description   string
 		serverSetup   func(string) (string, tearDown, error)
 		serverVersion string
-		request       *drapbv1alpha3.NodeUnprepareResourcesRequest
+		request       *drapb.NodeUnprepareResourcesRequest
 	}{
 		{
-			description:   "server supports v1alpha3",
+			description:   "server supports v1alpha4",
 			serverSetup:   setupFakeGRPCServer,
-			serverVersion: v1alpha3Version,
+			serverVersion: v1alpha4Version,
-			request:       &drapbv1alpha3.NodeUnprepareResourcesRequest{},
+			request:       &drapb.NodeUnprepareResourcesRequest{},
 		},
 	} {
 		t.Run(test.description, func(t *testing.T) {
+			ctx := ktesting.Init(t)
 			addr, teardown, err := setupFakeGRPCServer(test.serverVersion)
 			if err != nil {
 				t.Fatal(err)
 			}
 			defer teardown()
 
-			p := &plugin{
+			p := &Plugin{
+				backgroundCtx: ctx,
 				endpoint:      addr,
 				clientTimeout: PluginClientTimeout,
 			}
@@ -269,74 +267,3 @@ func TestNodeUnprepareResources(t *testing.T) {
 		})
 	}
 }
-
-func TestListAndWatchResources(t *testing.T) {
-	for _, test := range []struct {
-		description   string
-		serverSetup   func(string) (string, tearDown, error)
-		serverVersion string
-		request       *drapbv1alpha3.NodeListAndWatchResourcesRequest
-		responses     []*drapbv1alpha3.NodeListAndWatchResourcesResponse
-		expectError   string
-	}{
-		{
-			description:   "server supports NodeResources API",
-			serverSetup:   setupFakeGRPCServer,
-			serverVersion: v1alpha3Version,
-			request:       &drapbv1alpha3.NodeListAndWatchResourcesRequest{},
-			responses: []*drapbv1alpha3.NodeListAndWatchResourcesResponse{
-				{},
-				{},
-			},
-			expectError: "EOF",
-		},
-	} {
-		t.Run(test.description, func(t *testing.T) {
-			addr, teardown, err := setupFakeGRPCServer(test.serverVersion)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer teardown()
-
-			p := &plugin{
-				endpoint: addr,
-			}
-
-			conn, err := p.getOrCreateGRPCConn()
-			defer func() {
-				err := conn.Close()
-				if err != nil {
-					t.Error(err)
-				}
-			}()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			draPlugins.add("dummy-plugin", p)
-			defer draPlugins.delete("dummy-plugin")
-
-			client, err := NewDRAPluginClient("dummy-plugin")
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			stream, err := client.NodeListAndWatchResources(context.Background(), test.request)
-			if err != nil {
-				t.Fatal(err)
-			}
-			var actualResponses []*drapbv1alpha3.NodeListAndWatchResourcesResponse
-			var actualErr error
-			for {
-				resp, err := stream.Recv()
-				if err != nil {
-					actualErr = err
-					break
-				}
-				actualResponses = append(actualResponses, resp)
-			}
-			assert.Equal(t, test.responses, actualResponses)
-			assert.Contains(t, actualErr.Error(), test.expectError)
-		})
-	}
-}

pkg/kubelet/cm/dra/plugin/noderesources.go

@@ -1,520 +0,0 @@
-/*
-Copyright 2024 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package plugin
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"sync"
-	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-
-	v1 "k8s.io/api/core/v1"
-	resourceapi "k8s.io/api/resource/v1alpha2"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/sets"
-	resourceinformers "k8s.io/client-go/informers/resource/v1alpha2"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/util/flowcontrol"
-	"k8s.io/client-go/util/workqueue"
-	"k8s.io/klog/v2"
-	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha3"
-	"k8s.io/utils/ptr"
-)
-
-const (
-	// resyncPeriod for informer
-	// TODO (https://github.com/kubernetes/kubernetes/issues/123688): disable?
-	resyncPeriod   = time.Duration(10 * time.Minute)
-	retryPeriod    = 5 * time.Second
-	maxRetryPeriod = 180 * time.Second
-	backoffFactor  = 2.0 // Introduce a backoff multiplier as jitter factor
-)
-
-// nodeResourcesController collects resource information from all registered
-// plugins and synchronizes that information with ResourceSlice objects.
-type nodeResourcesController struct {
-	ctx        context.Context
-	kubeClient kubernetes.Interface
-	getNode    func() (*v1.Node, error)
-	wg         sync.WaitGroup
-	queue      workqueue.TypedRateLimitingInterface[string]
-	sliceStore cache.Store
-
-	mutex         sync.RWMutex
-	activePlugins map[string]*activePlugin
-}
-
-// activePlugin holds the resource information about one plugin
-// and the gRPC stream that is used to retrieve that. The context
-// used by that stream can be canceled separately to stop
-// the monitoring.
-type activePlugin struct {
-	// cancel is the function which cancels the monitorPlugin goroutine
-	// for this plugin.
-	cancel func(reason error)
-
-	// resources is protected by the nodeResourcesController read/write lock.
-	// When receiving updates from the driver, the entire slice gets replaced,
-	// so it is okay to not do a deep copy of it. Only retrieving the slice
-	// must be protected by a read lock.
-	resources []*resourceapi.ResourceModel
-}
-
-// startNodeResourcesController constructs a new controller and starts it.
-//
-// If a kubeClient is provided, then it synchronizes ResourceSlices
-// with the resource information provided by plugins. Without it,
-// the controller is inactive. This can happen when kubelet is run stand-alone
-// without an apiserver. In that case we can't and don't need to publish
-// ResourceSlices.
-func startNodeResourcesController(ctx context.Context, kubeClient kubernetes.Interface, getNode func() (*v1.Node, error)) *nodeResourcesController {
-	if kubeClient == nil {
-		return nil
-	}
-
-	logger := klog.FromContext(ctx)
-	logger = klog.LoggerWithName(logger, "node resources controller")
-	ctx = klog.NewContext(ctx, logger)
-
-	c := &nodeResourcesController{
-		ctx:        ctx,
-		kubeClient: kubeClient,
-		getNode:    getNode,
-		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
-			workqueue.DefaultTypedControllerRateLimiter[string](),
-			workqueue.TypedRateLimitingQueueConfig[string]{Name: "node_resource_slices"},
-		),
-		activePlugins: make(map[string]*activePlugin),
-	}
-
-	c.wg.Add(1)
-	go func() {
-		defer c.wg.Done()
-		c.run(ctx)
-	}()
-
-	return c
-}
-
-// waitForStop blocks until all background activity spawned by
-// the controller has stopped. The context passed to start must
-// be canceled for that to happen.
-//
-// Not needed at the moment, but if it was, this is what it would
-// look like...
-// func (c *nodeResourcesController) waitForStop() {
-// 	if c == nil {
-// 		return
-// 	}
-//
-// 	c.wg.Wait()
-// }
-
-// addPlugin is called whenever a plugin has been (re-)registered.
-func (c *nodeResourcesController) addPlugin(driverName string, pluginInstance *plugin) {
-	if c == nil {
-		return
-	}
-
-	klog.FromContext(c.ctx).V(2).Info("Adding plugin", "driverName", driverName)
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-
-	if active := c.activePlugins[driverName]; active != nil {
-		active.cancel(errors.New("plugin has re-registered"))
-	}
-	active := &activePlugin{}
-	cancelCtx, cancel := context.WithCancelCause(c.ctx)
-	active.cancel = cancel
-	c.activePlugins[driverName] = active
-	c.queue.Add(driverName)
-
-	c.wg.Add(1)
-	go func() {
-		defer c.wg.Done()
-		c.monitorPlugin(cancelCtx, active, driverName, pluginInstance)
-	}()
-}
-
-// removePlugin is called whenever a plugin has been unregistered.
-func (c *nodeResourcesController) removePlugin(driverName string) {
-	if c == nil {
-		return
-	}
-
-	klog.FromContext(c.ctx).V(2).Info("Removing plugin", "driverName", driverName)
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-	if active, ok := c.activePlugins[driverName]; ok {
-		active.cancel(errors.New("plugin has unregistered"))
-		delete(c.activePlugins, driverName)
-		c.queue.Add(driverName)
-	}
-}
-
-// monitorPlugin calls the plugin to retrieve resource information and caches
-// all responses that it gets for processing in the sync method. It keeps
-// retrying until an error or EOF response indicates that no further data is
-// going to be sent, then watch resources of the plugin stops until it
-// re-registers.
-func (c *nodeResourcesController) monitorPlugin(ctx context.Context, active *activePlugin, driverName string, pluginInstance *plugin) {
-	logger := klog.FromContext(ctx)
-	logger = klog.LoggerWithValues(logger, "driverName", driverName)
-	logger.Info("Starting to monitor node resources of the plugin")
-	defer func() {
-		r := recover()
-		logger.Info("Stopping to monitor node resources of the plugin", "reason", context.Cause(ctx), "err", ctx.Err(), "recover", r)
-	}()
-
-	backOff := flowcontrol.NewBackOffWithJitter(retryPeriod, maxRetryPeriod, backoffFactor)
-	backOffID := "retry"
-
-	// Keep trying until canceled.
-	for ctx.Err() == nil {
-		logger.V(5).Info("Calling NodeListAndWatchResources")
-		stream, err := pluginInstance.NodeListAndWatchResources(ctx, new(drapb.NodeListAndWatchResourcesRequest))
-		if err != nil {
-			switch {
-			case status.Convert(err).Code() == codes.Unimplemented:
-				// The plugin simply doesn't provide node resources.
-				active.cancel(errors.New("plugin does not support node resource reporting"))
-			default:
-				// This is a problem, report it and retry.
-				logger.Error(err, "Creating gRPC stream for node resources failed")
-				select {
-				case <-time.After(backOff.Get(backOffID)):
-					backOff.Next(backOffID, time.Now())
-				case <-ctx.Done():
-				}
-			}
-			continue
-		}
-		for {
-			response, err := stream.Recv()
-			if err != nil {
-				switch {
-				case errors.Is(err, io.EOF):
-					// This is okay. Some plugins might never change their
-					// resources after reporting them once.
-					active.cancel(errors.New("plugin has closed the stream"))
-				case status.Convert(err).Code() == codes.Unimplemented:
-					// The plugin has the method, does not really implement it.
-					active.cancel(errors.New("plugin does not support node resource reporting"))
-				case ctx.Err() == nil:
-					// This is a problem, report it and retry.
-					logger.Error(err, "Reading node resources from gRPC stream failed")
-					select {
-					case <-time.After(backOff.Get(backOffID)):
-						backOff.Next(backOffID, time.Now())
-					case <-ctx.Done():
-					}
-				}
-				break
-			}
-
-			if loggerV := logger.V(6); loggerV.Enabled() {
-				loggerV.Info("Driver resources updated", "resources", response.Resources)
-			} else {
-				logger.V(5).Info("Driver resources updated", "numResources", len(response.Resources))
-			}
-
-			c.mutex.Lock()
-			active.resources = response.Resources
-			c.mutex.Unlock()
-			c.queue.Add(driverName)
-		}
-	}
-}
-
-// run is running in the background. It handles blocking initialization (like
-// syncing the informer) and then syncs the actual with the desired state.
-func (c *nodeResourcesController) run(ctx context.Context) {
-	logger := klog.FromContext(ctx)
-
-	// When kubelet starts, we have two choices:
-	// - Sync immediately, which in practice will delete all ResourceSlices
-	//   because no plugin has registered yet. We could do a DeleteCollection
-	//   to speed this up.
-	// - Wait a bit, then sync. If all plugins have re-registered in the meantime,
-	//   we might not need to change any ResourceSlice.
-	//
-	// For now syncing starts immediately, with no DeleteCollection. This
-	// can be reconsidered later.
-
-	// Wait until we're able to get a Node object.
-	// This means that the object is created on the API server,
-	// the kubeclient is functional and the node informer cache is populated with the node object.
-	// Without this it doesn't make sense to proceed further as we need a node name and
-	// a node UID for this controller to work.
-	var node *v1.Node
-	var err error
-	for {
-		node, err = c.getNode()
-		if err == nil {
-			break
-		}
-		logger.V(5).Info("Getting Node object failed, waiting", "err", err)
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(time.Second):
-		}
-	}
-
-	// We could use an indexer on driver name, but that seems overkill.
-	informer := resourceinformers.NewFilteredResourceSliceInformer(c.kubeClient, resyncPeriod, nil, func(options *metav1.ListOptions) {
-		options.FieldSelector = "nodeName=" + node.Name
-	})
-	c.sliceStore = informer.GetStore()
-	handler, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc: func(obj any) {
-			slice, ok := obj.(*resourceapi.ResourceSlice)
-			if !ok {
-				return
-			}
-			logger.V(5).Info("ResourceSlice add", "slice", klog.KObj(slice))
-			c.queue.Add(slice.DriverName)
-		},
-		UpdateFunc: func(old, new any) {
-			oldSlice, ok := old.(*resourceapi.ResourceSlice)
-			if !ok {
-				return
-			}
-			newSlice, ok := new.(*resourceapi.ResourceSlice)
-			if !ok {
-				return
-			}
-			if loggerV := logger.V(6); loggerV.Enabled() {
-				loggerV.Info("ResourceSlice update", "slice", klog.KObj(newSlice), "diff", cmp.Diff(oldSlice, newSlice))
-			} else {
-				logger.V(5).Info("ResourceSlice update", "slice", klog.KObj(newSlice))
-			}
-			c.queue.Add(newSlice.DriverName)
-		},
-		DeleteFunc: func(obj any) {
-			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
-				obj = tombstone.Obj
-			}
-			slice, ok := obj.(*resourceapi.ResourceSlice)
-			if !ok {
-				return
-			}
-			logger.V(5).Info("ResourceSlice delete", "slice", klog.KObj(slice))
-			c.queue.Add(slice.DriverName)
-		},
-	})
-	if err != nil {
-		logger.Error(err, "Registering event handler on the ResourceSlice informer failed, disabling resource monitoring")
-		return
-	}
-
-	// Start informer and wait for our cache to be populated.
-	c.wg.Add(1)
-	go func() {
-		defer c.wg.Done()
-		informer.Run(ctx.Done())
-	}()
-	for !handler.HasSynced() {
-		select {
-		case <-time.After(time.Second):
-		case <-ctx.Done():
-			return
-		}
-	}
-	logger.Info("ResourceSlice informer has synced")
-
-	for c.processNextWorkItem(ctx) {
-	}
-}
-
-func (c *nodeResourcesController) processNextWorkItem(ctx context.Context) bool {
-	key, shutdown := c.queue.Get()
-	if shutdown {
-		return false
-	}
-	defer c.queue.Done(key)
-
-	driverName := key
-
-	// Panics are caught and treated like errors.
-	var err error
-	func() {
-		defer func() {
-			if r := recover(); r != nil {
-				err = fmt.Errorf("internal error: %v", r)
-			}
-		}()
-		err = c.sync(ctx, driverName)
-	}()
-
-	if err != nil {
-		// TODO (https://github.com/kubernetes/enhancements/issues/3077): contextual logging in utilruntime
-		utilruntime.HandleError(fmt.Errorf("processing driver %v: %v", driverName, err))
-		c.queue.AddRateLimited(key)
-
-		// Return without removing the work item from the queue.
-		// It will be retried.
-		return true
-	}
-
-	c.queue.Forget(key)
-	return true
-}
-
-func (c *nodeResourcesController) sync(ctx context.Context, driverName string) error {
-	logger := klog.FromContext(ctx)
-
-	// Gather information about the actual and desired state.
-	slices := c.sliceStore.List()
-	var driverResources []*resourceapi.ResourceModel
-	c.mutex.RLock()
-	if active, ok := c.activePlugins[driverName]; ok {
-		// No need for a deep copy, the entire slice gets replaced on writes.
-		driverResources = active.resources
-	}
-	c.mutex.RUnlock()
-
-	// Resources that are not yet stored in any slice need to be published.
-	// Here we track the indices of any resources that are already stored.
-	storedResourceIndices := sets.New[int]()
-
-	// Slices that don't match any driver resource can either be updated (if there
-	// are new driver resources that need to be stored) or they need to be deleted.
-	obsoleteSlices := make([]*resourceapi.ResourceSlice, 0, len(slices))
-
-	// Match slices with resource information.
-	for _, obj := range slices {
-		slice := obj.(*resourceapi.ResourceSlice)
-		if slice.DriverName != driverName {
-			continue
-		}
-
-		index := indexOfModel(driverResources, &slice.ResourceModel)
-		if index >= 0 {
-			storedResourceIndices.Insert(index)
-			continue
-		}
-
-		obsoleteSlices = append(obsoleteSlices, slice)
-	}
-
-	if loggerV := logger.V(6); loggerV.Enabled() {
-		// Dump entire resource information.
-		loggerV.Info("Syncing existing driver node resource slices with driver resources", "slices", klog.KObjSlice(slices), "resources", driverResources)
-	} else {
-		logger.V(5).Info("Syncing existing driver node resource slices with driver resources", "slices", klog.KObjSlice(slices), "numResources", len(driverResources))
-	}
-
-	// Update stale slices before removing what's left.
-	//
-	// We don't really know which of these slices might have
-	// been used for "the" driver resource because they don't
-	// have a unique ID. In practice, a driver is most likely
-	// to just give us one ResourceModel, in which case
-	// this isn't a problem at all. If we have more than one,
-	// then at least conceptually it currently doesn't matter
-	// where we publish it.
-	//
-	// The long-term goal is to move the handling of
-	// ResourceSlice objects into the driver, with kubelet
-	// just acting as a REST proxy. The advantage of that will
-	// be that kubelet won't need to support the same
-	// resource API version as the driver and the control plane.
-	// With that approach, the driver will be able to match
-	// up objects more intelligently.
-	numObsoleteSlices := len(obsoleteSlices)
-	for index, resource := range driverResources {
-		if storedResourceIndices.Has(index) {
-			// No need to do anything, it is already stored exactly
-			// like this in an existing slice.
-			continue
-		}
-
-		if numObsoleteSlices > 0 {
-			// Update one existing slice.
-			slice := obsoleteSlices[numObsoleteSlices-1]
-			numObsoleteSlices--
-			slice = slice.DeepCopy()
-			slice.ResourceModel = *resource
-			logger.V(5).Info("Reusing existing node resource slice", "slice", klog.KObj(slice))
-			if _, err := c.kubeClient.ResourceV1alpha2().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}); err != nil {
-				return fmt.Errorf("update node resource slice: %w", err)
-			}
-			continue
-		}
-
-		// Although node name and UID are unlikely to change
-		// we're getting updated node object just to be on the safe side.
-		// It's a cheap operation as it gets an object from the node informer cache.
-		node, err := c.getNode()
-		if err != nil {
-			return fmt.Errorf("retrieve node object: %w", err)
-		}
-
-		// Create a new slice.
-		slice := &resourceapi.ResourceSlice{
-			ObjectMeta: metav1.ObjectMeta{
-				GenerateName: node.Name + "-" + driverName + "-",
-				OwnerReferences: []metav1.OwnerReference{
-					{
-						APIVersion: v1.SchemeGroupVersion.WithKind("Node").Version,
-						Kind:       v1.SchemeGroupVersion.WithKind("Node").Kind,
-						Name:       node.Name,
-						UID:        node.UID,
-						Controller: ptr.To(true),
-					},
-				},
-			},
-			NodeName:      node.Name,
-			DriverName:    driverName,
-			ResourceModel: *resource,
-		}
-		logger.V(5).Info("Creating new node resource slice", "slice", klog.KObj(slice))
-		if _, err := c.kubeClient.ResourceV1alpha2().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}); err != nil {
-			return fmt.Errorf("create node resource slice: %w", err)
-		}
-	}
-
-	// All remaining slices are truly orphaned.
-	for i := 0; i < numObsoleteSlices; i++ {
-		slice := obsoleteSlices[i]
-		logger.V(5).Info("Deleting obsolete node resource slice", "slice", klog.KObj(slice))
-		if err := c.kubeClient.ResourceV1alpha2().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{}); err != nil {
-			return fmt.Errorf("delete node resource slice: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func indexOfModel(models []*resourceapi.ResourceModel, model *resourceapi.ResourceModel) int {
-	for index, m := range models {
-		if apiequality.Semantic.DeepEqual(m, model) {
-			return index
-		}
-	}
-	return -1
-}

pkg/kubelet/cm/dra/plugin/plugin.go

@@ -20,94 +20,125 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"strings"
-	"sync"
 	"time"
 
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/credentials/insecure"
 	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/kubelet/pluginmanager/cache"
 )
 
-const (
-	// DRAPluginName is the name of the in-tree DRA Plugin.
-	DRAPluginName   = "kubernetes.io/dra"
-	v1alpha3Version = "v1alpha3"
-)
-
-// Plugin is a description of a DRA Plugin, defined by an endpoint.
-type plugin struct {
-	sync.Mutex
-	conn                    *grpc.ClientConn
-	endpoint                string
-	highestSupportedVersion *utilversion.Version
-	clientTimeout           time.Duration
-}
-
-func (p *plugin) getOrCreateGRPCConn() (*grpc.ClientConn, error) {
-	p.Lock()
-	defer p.Unlock()
-
-	if p.conn != nil {
-		return p.conn, nil
-	}
-
-	network := "unix"
-	klog.V(4).InfoS(log("creating new gRPC connection"), "protocol", network, "endpoint", p.endpoint)
-	conn, err := grpc.Dial(
-		p.endpoint,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) {
-			return (&net.Dialer{}).DialContext(ctx, network, target)
-		}),
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-
-	if ok := conn.WaitForStateChange(ctx, connectivity.Connecting); !ok {
-		return nil, errors.New("timed out waiting for gRPC connection to be ready")
-	}
-
-	p.conn = conn
-	return p.conn, nil
-}
-
 // RegistrationHandler is the handler which is fed to the pluginwatcher API.
 type RegistrationHandler struct {
-	controller *nodeResourcesController
+	// backgroundCtx is used for all future activities of the handler.
+	// This is necessary because it implements APIs which don't
+	// provide a context.
+	backgroundCtx context.Context
+	kubeClient    kubernetes.Interface
+	getNode       func() (*v1.Node, error)
 }
 
+var _ cache.PluginHandler = &RegistrationHandler{}
+
 // NewPluginHandler returns new registration handler.
 //
 // Must only be called once per process because it manages global state.
 // If a kubeClient is provided, then it synchronizes ResourceSlices
 // with the resource information provided by plugins.
 func NewRegistrationHandler(kubeClient kubernetes.Interface, getNode func() (*v1.Node, error)) *RegistrationHandler {
-	handler := &RegistrationHandler{}
+	handler := &RegistrationHandler{
+		// The context and thus logger should come from the caller.
+		backgroundCtx: klog.NewContext(context.TODO(), klog.LoggerWithName(klog.TODO(), "DRA registration handler")),
+		kubeClient:    kubeClient,
+		getNode:       getNode,
+	}
 
-	// If kubelet ever gets an API for stopping registration handlers, then
+	// When kubelet starts up, no DRA driver has registered yet. None of
-	// that would need to be hooked up with stopping the controller.
+	// the drivers are usable until they come back, which might not happen
-	handler.controller = startNodeResourcesController(context.TODO(), kubeClient, getNode)
+	// at all. Therefore it is better to not advertise any local resources
+	// because pods could get stuck on the node waiting for the driver
+	// to start up.
+	//
+	// This has to run in the background.
+	go handler.wipeResourceSlices("")
 
 	return handler
 }
 
+// wipeResourceSlices deletes ResourceSlices of the node, optionally just for a specific driver.
+func (h *RegistrationHandler) wipeResourceSlices(pluginName string) {
+	if h.kubeClient == nil {
+		return
+	}
+	ctx := h.backgroundCtx
+	logger := klog.FromContext(ctx)
+
+	backoff := wait.Backoff{
+		Duration: time.Second,
+		Factor:   2,
+		Jitter:   0.2,
+		Cap:      5 * time.Minute,
+		Steps:    100,
+	}
+
+	// Error logging is done inside the loop. Context cancellation doesn't get logged.
+	_ = wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) {
+		node, err := h.getNode()
+		if apierrors.IsNotFound(err) {
+			return false, nil
+		}
+		if err != nil {
+			logger.Error(err, "Unexpected error checking for node")
+			return false, nil
+		}
+		fieldSelector := fields.Set{"nodeName": node.Name}
+		if pluginName != "" {
+			fieldSelector["driverName"] = pluginName
+		}
+
+		err = h.kubeClient.ResourceV1alpha2().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: fieldSelector.String()})
+		switch {
+		case err == nil:
+			logger.V(3).Info("Deleted ResourceSlices", "fieldSelector", fieldSelector)
+			return true, nil
+		case apierrors.IsUnauthorized(err):
+			// This can happen while kubelet is still figuring out
+			// its credentials.
+			logger.V(5).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err)
+			return false, nil
+		default:
+			// Log and retry for other errors.
+			logger.V(3).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err)
+			return false, nil
+		}
+	})
+}
+
 // RegisterPlugin is called when a plugin can be registered.
 func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string, versions []string, pluginClientTimeout *time.Duration) error {
-	klog.InfoS("Register new DRA plugin", "name", pluginName, "endpoint", endpoint)
+	// Prepare a context with its own logger for the plugin.
+	//
+	// The lifecycle of the plugin's background activities is tied to our
+	// root context, so canceling that will also cancel the plugin.
+	//
+	// The logger injects the plugin name as additional value
+	// into all log output related to the plugin.
+	ctx := h.backgroundCtx
+	logger := klog.FromContext(ctx)
+	logger = klog.LoggerWithValues(logger, "pluginName", pluginName)
+	ctx = klog.NewContext(ctx, logger)
 
-	highestSupportedVersion, err := h.validateVersions("RegisterPlugin", pluginName, versions)
+	logger.V(3).Info("Register new DRA plugin", "endpoint", endpoint)
+
+	highestSupportedVersion, err := h.validateVersions(pluginName, versions)
 	if err != nil {
-		return err
+		return fmt.Errorf("version check of plugin %s failed: %w", pluginName, err)
 	}
 
 	var timeout time.Duration
@@ -117,7 +148,11 @@ func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string,
 		timeout = *pluginClientTimeout
 	}
 
-	pluginInstance := &plugin{
+	ctx, cancel := context.WithCancelCause(ctx)
+
+	pluginInstance := &Plugin{
+		backgroundCtx:           ctx,
+		cancel:                  cancel,
 		conn:                    nil,
 		endpoint:                endpoint,
 		highestSupportedVersion: highestSupportedVersion,
@@ -126,40 +161,27 @@ func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string,
 
 	// Storing endpoint of newly registered DRA Plugin into the map, where plugin name will be the key
 	// all other DRA components will be able to get the actual socket of DRA plugins by its name.
-	// By default we assume the supported plugin version is v1alpha3
+	if draPlugins.add(pluginName, pluginInstance) {
-	draPlugins.add(pluginName, pluginInstance)
+		logger.V(1).Info("Already registered, previous plugin was replaced")
-	h.controller.addPlugin(pluginName, pluginInstance)
+	}
 
 	return nil
 }
 
 func (h *RegistrationHandler) validateVersions(
-	callerName string,
 	pluginName string,
 	versions []string,
 ) (*utilversion.Version, error) {
 	if len(versions) == 0 {
-		return nil, errors.New(
+		return nil, errors.New("empty list for supported versions")
-			log(
-				"%s for DRA plugin %q failed. Plugin returned an empty list for supported versions",
-				callerName,
-				pluginName,
-			),
-		)
 	}
 
 	// Validate version
 	newPluginHighestVersion, err := utilversion.HighestSupportedVersion(versions)
 	if err != nil {
-		return nil, errors.New(
+		// HighestSupportedVersion includes the list of versions in its error
-			log(
+		// if relevant, no need to repeat it here.
-				"%s for DRA plugin %q failed. None of the versions specified %q are supported. err=%v",
+		return nil, fmt.Errorf("none of the versions are supported: %w", err)
-				callerName,
-				pluginName,
-				versions,
-				err,
-			),
-		)
 	}
 
 	existingPlugin := draPlugins.get(pluginName)
@@ -169,26 +191,26 @@ func (h *RegistrationHandler) validateVersions(
 	if existingPlugin.highestSupportedVersion.LessThan(newPluginHighestVersion) {
 		return newPluginHighestVersion, nil
 	}
-	return nil, errors.New(
+	return nil, fmt.Errorf("another plugin instance is already registered with a higher supported version: %q < %q", newPluginHighestVersion, existingPlugin.highestSupportedVersion)
-		log(
-			"%s for DRA plugin %q failed. Another plugin with the same name is already registered with a higher supported version: %q",
-			callerName,
-			pluginName,
-			existingPlugin.highestSupportedVersion,
-		),
-	)
-}
-
-func deregisterPlugin(pluginName string) {
-	draPlugins.delete(pluginName)
 }
 
 // DeRegisterPlugin is called when a plugin has removed its socket,
 // signaling it is no longer available.
 func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) {
-	klog.InfoS("DeRegister DRA plugin", "name", pluginName)
+	if p := draPlugins.delete(pluginName); p != nil {
-	deregisterPlugin(pluginName)
+		logger := klog.FromContext(p.backgroundCtx)
-	h.controller.removePlugin(pluginName)
+		logger.V(3).Info("Deregister DRA plugin", "endpoint", p.endpoint)
+
+		// Clean up the ResourceSlices for the deleted Plugin since it
+		// may have died without doing so itself and might never come
+		// back.
+		go h.wipeResourceSlices(pluginName)
+
+		return
+	}
+
+	logger := klog.FromContext(h.backgroundCtx)
+	logger.V(3).Info("Deregister DRA plugin not necessary, was already removed")
 }
 
 // ValidatePlugin is called by kubelet's plugin watcher upon detection
@@ -196,15 +218,10 @@ func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) {
 func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error {
 	klog.InfoS("Validate DRA plugin", "name", pluginName, "endpoint", endpoint, "versions", strings.Join(versions, ","))
 
-	_, err := h.validateVersions("ValidatePlugin", pluginName, versions)
+	_, err := h.validateVersions(pluginName, versions)
 	if err != nil {
-		return fmt.Errorf("validation failed for DRA plugin %s at endpoint %s: %+v", pluginName, endpoint, err)
+		return fmt.Errorf("invalid versions of plugin %s: %w", pluginName, err)
 	}
 
 	return err
 }
-
-// log prepends log string with `kubernetes.io/dra`.
-func log(msg string, parts ...interface{}) string {
-	return fmt.Sprintf(fmt.Sprintf("%s: %s", DRAPluginName, msg), parts...)
-}

pkg/kubelet/cm/dra/plugin/plugins_store.go

@@ -17,15 +17,14 @@ limitations under the License.
 package plugin
 
 import (
+	"errors"
 	"sync"
-
-	"k8s.io/klog/v2"
 )
 
 // PluginsStore holds a list of DRA Plugins.
 type pluginsStore struct {
 	sync.RWMutex
-	store map[string]*plugin
+	store map[string]*Plugin
 }
 
 // draPlugins map keeps track of all registered DRA plugins on the node
@@ -34,7 +33,7 @@ var draPlugins = &pluginsStore{}
 
 // Get lets you retrieve a DRA Plugin by name.
 // This method is protected by a mutex.
-func (s *pluginsStore) get(pluginName string) *plugin {
+func (s *pluginsStore) get(pluginName string) *Plugin {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -43,26 +42,33 @@ func (s *pluginsStore) get(pluginName string) *plugin {
 
 // Set lets you save a DRA Plugin to the list and give it a specific name.
 // This method is protected by a mutex.
-func (s *pluginsStore) add(pluginName string, p *plugin) {
+func (s *pluginsStore) add(pluginName string, p *Plugin) (replaced bool) {
 	s.Lock()
 	defer s.Unlock()
 
 	if s.store == nil {
-		s.store = make(map[string]*plugin)
+		s.store = make(map[string]*Plugin)
 	}
 
 	_, exists := s.store[pluginName]
-	if exists {
-		klog.V(1).InfoS(log("plugin: %s already registered, previous plugin will be overridden", pluginName))
-	}
 	s.store[pluginName] = p
+	return exists
 }
 
 // Delete lets you delete a DRA Plugin by name.
 // This method is protected by a mutex.
-func (s *pluginsStore) delete(pluginName string) {
+func (s *pluginsStore) delete(pluginName string) *Plugin {
 	s.Lock()
 	defer s.Unlock()
 
+	p, exists := s.store[pluginName]
+	if !exists {
+		return nil
+	}
+	if p.cancel != nil {
+		p.cancel(errors.New("plugin got removed"))
+	}
 	delete(s.store, pluginName)
+
+	return p
 }

plugin/pkg/admission/noderestriction/admission.go

@@ -641,8 +641,14 @@ func (p *Plugin) admitCSINode(nodeName string, a admission.Attributes) error {
 
 func (p *Plugin) admitResourceSlice(nodeName string, a admission.Attributes) error {
 	// The create request must come from a node with the same name as the NodeName field.
-	// Other requests gets checked by the node authorizer.
+	// Same when deleting an object.
-	if a.GetOperation() == admission.Create {
+	//
+	// Other requests get checked by the node authorizer. The checks here are necessary
+	// because the node authorizer does not know the object content for a create request
+	// and not each deleted object in a DeleteCollection. DeleteCollection checks each
+	// individual object.
+	switch a.GetOperation() {
+	case admission.Create:
 		slice, ok := a.GetObject().(*resource.ResourceSlice)
 		if !ok {
 			return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject()))
@@ -651,6 +657,15 @@ func (p *Plugin) admitResourceSlice(nodeName string, a admission.Attributes) err
 		if slice.NodeName != nodeName {
 			return admission.NewForbidden(a, errors.New("can only create ResourceSlice with the same NodeName as the requesting node"))
 		}
+	case admission.Delete:
+		slice, ok := a.GetOldObject().(*resource.ResourceSlice)
+		if !ok {
+			return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetOldObject()))
+		}
+
+		if slice.NodeName != nodeName {
+			return admission.NewForbidden(a, errors.New("can only delete ResourceSlice with the same NodeName as the requesting node"))
+		}
 	}
 
 	return nil

plugin/pkg/admission/noderestriction/admission_test.go

@@ -1607,7 +1607,8 @@ func TestAdmitResourceSlice(t *testing.T) {
 	apiResource := resourceapi.SchemeGroupVersion.WithResource("resourceslices")
 	nodename := "mynode"
 	mynode := &user.DefaultInfo{Name: "system:node:" + nodename, Groups: []string{"system:nodes"}}
-	err := "can only create ResourceSlice with the same NodeName as the requesting node"
+	createErr := "can only create ResourceSlice with the same NodeName as the requesting node"
+	deleteErr := "can only delete ResourceSlice with the same NodeName as the requesting node"
 
 	sliceNode := &resourceapi.ResourceSlice{
 		ObjectMeta: metav1.ObjectMeta{
@@ -1624,53 +1625,88 @@ func TestAdmitResourceSlice(t *testing.T) {
 
 	tests := map[string]struct {
 		operation      admission.Operation
-		obj            runtime.Object
+		options        runtime.Object
+		obj, oldObj    runtime.Object
 		featureEnabled bool
 		expectError    string
 	}{
 		"create allowed, enabled": {
 			operation:      admission.Create,
+			options:        &metav1.CreateOptions{},
 			obj:            sliceNode,
 			featureEnabled: true,
 			expectError:    "",
 		},
 		"create disallowed, enabled": {
 			operation:      admission.Create,
+			options:        &metav1.CreateOptions{},
 			obj:            sliceOtherNode,
 			featureEnabled: true,
-			expectError:    err,
+			expectError:    createErr,
 		},
 		"create allowed, disabled": {
 			operation:      admission.Create,
+			options:        &metav1.CreateOptions{},
 			obj:            sliceNode,
 			featureEnabled: false,
 			expectError:    "",
 		},
 		"create disallowed, disabled": {
 			operation:      admission.Create,
+			options:        &metav1.CreateOptions{},
 			obj:            sliceOtherNode,
 			featureEnabled: false,
-			expectError:    err,
+			expectError:    createErr,
 		},
 		"update allowed, same node": {
 			operation:      admission.Update,
+			options:        &metav1.UpdateOptions{},
 			obj:            sliceNode,
 			featureEnabled: true,
 			expectError:    "",
 		},
 		"update allowed, other node": {
 			operation:      admission.Update,
+			options:        &metav1.UpdateOptions{},
 			obj:            sliceOtherNode,
 			featureEnabled: true,
 			expectError:    "",
 		},
+		"delete allowed, enabled": {
+			operation:      admission.Delete,
+			options:        &metav1.DeleteOptions{},
+			oldObj:         sliceNode,
+			featureEnabled: true,
+			expectError:    "",
+		},
+		"delete disallowed, enabled": {
+			operation:      admission.Delete,
+			options:        &metav1.DeleteOptions{},
+			oldObj:         sliceOtherNode,
+			featureEnabled: true,
+			expectError:    deleteErr,
+		},
+		"delete allowed, disabled": {
+			operation:      admission.Delete,
+			options:        &metav1.DeleteOptions{},
+			oldObj:         sliceNode,
+			featureEnabled: false,
+			expectError:    "",
+		},
+		"delete disallowed, disabled": {
+			operation:      admission.Delete,
+			options:        &metav1.DeleteOptions{},
+			oldObj:         sliceOtherNode,
+			featureEnabled: false,
+			expectError:    deleteErr,
+		},
 	}
 
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
 			attributes := admission.NewAttributesRecord(
-				test.obj, nil, schema.GroupVersionKind{},
+				test.obj, test.oldObj, schema.GroupVersionKind{},
-				"", "foo", apiResource, "", test.operation, &metav1.CreateOptions{}, false, mynode)
+				"", "foo", apiResource, "", test.operation, test.options, false, mynode)
 			featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.DynamicResourceAllocation, test.featureEnabled)
 			a := &admitTestCase{
 				name:       name,

plugin/pkg/auth/authorizer/node/node_authorizer.go

@@ -309,30 +309,34 @@ func (r *NodeAuthorizer) authorizeResourceSlice(nodeName string, attrs authorize
 		return authorizer.DecisionNoOpinion, "cannot authorize ResourceSlice subresources", nil
 	}
 
-	// allowed verbs: get, create, update, patch, delete
+	// allowed verbs: get, create, update, patch, delete, watch, list, deletecollection
 	verb := attrs.GetVerb()
 	switch verb {
-	case "get", "create", "update", "patch", "delete":
+	case "create":
-		// Okay, but check individual object permission below.
+		// The request must come from a node with the same name as the ResourceSlice.NodeName field.
-	case "watch", "list":
+		//
-		// Okay. The kubelet is trusted to use a filter for its own objects.
+		// For create, the noderestriction admission plugin is performing this check.
+		// Here we don't have access to the content of the new object.
+		return authorizer.DecisionAllow, "", nil
+	case "get", "update", "patch", "delete":
+		// Checking the existing object must have established that access
+		// is allowed by recording a graph edge.
+		return r.authorize(nodeName, sliceVertexType, attrs)
+	case "watch", "list", "deletecollection":
+		// Okay. The kubelet is trusted to use a filter for its own objects in watch and list.
+		// The NodeRestriction admission plugin (plugin/pkg/admission/noderestriction)
+		// ensures that the node is not deleting some ResourceSlice belonging to
+		// some other node.
+		//
+		// TODO (https://github.com/kubernetes/kubernetes/issues/125355):
+		// Once https://github.com/kubernetes/enhancements/pull/4600 is implemented,
+		// this code needs to be extended to verify that the node filter is indeed set.
+		// Then the admission check can be removed.
 		return authorizer.DecisionAllow, "", nil
 	default:
 		klog.V(2).Infof("NODE DENY: '%s' %#v", nodeName, attrs)
-		return authorizer.DecisionNoOpinion, "can only get, create, update, patch, or delete a ResourceSlice", nil
+		return authorizer.DecisionNoOpinion, "only the following verbs are allowed for a ResourceSlice: get, watch, list, create, update, patch, delete, deletecollection", nil
 	}
-
-	// The request must come from a node with the same name as the ResourceSlice.NodeName field.
-	//
-	// For create, the noderestriction admission plugin is performing this check.
-	// Here we don't have access to the content of the new object.
-	if verb == "create" {
-		return authorizer.DecisionAllow, "", nil
-	}
-
-	// For any other verb, checking the existing object must have established that access
-	// is allowed by recording a graph edge.
-	return r.authorize(nodeName, sliceVertexType, attrs)
 }
 
 // hasPathFrom returns true if there is a directed path from the specified type/namespace/name to the specified Node

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

@@ -181,6 +181,7 @@ func NodeRules() []rbacv1.PolicyRule {
 	// DRA Resource Claims
 	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicResourceAllocation) {
 		nodePolicyRules = append(nodePolicyRules, rbacv1helpers.NewRule("get").Groups(resourceGroup).Resources("resourceclaims").RuleOrDie())
+		nodePolicyRules = append(nodePolicyRules, rbacv1helpers.NewRule("deletecollection").Groups(resourceGroup).Resources("resourceslices").RuleOrDie())
 	}
 	// Kubelet needs access to ClusterTrustBundles to support the pemTrustAnchors volume type.
 	if utilfeature.DefaultFeatureGate.Enabled(features.ClusterTrustBundle) {

staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/draplugin.go

@@ -17,14 +17,20 @@ limitations under the License.
 package kubeletplugin
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net"
+	"sync"
 
 	"google.golang.org/grpc"
 	"k8s.io/klog/v2"
 
-	drapbv1alpha3 "k8s.io/kubelet/pkg/apis/dra/v1alpha3"
+	resourceapi "k8s.io/api/resource/v1alpha2"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/dynamic-resource-allocation/resourceslice"
+	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4"
 	registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1"
 )
 
@@ -39,6 +45,18 @@ type DRAPlugin interface {
 	// received yet.
 	RegistrationStatus() *registerapi.RegistrationStatus
 
+	// PublishResources may be called one or more times to publish
+	// resource information in ResourceSlice objects. If it never gets
+	// called, then the kubelet plugin does not manage any ResourceSlice
+	// objects.
+	//
+	// PublishResources does not block, so it might still take a while
+	// after it returns before all information is actually written
+	// to the API server.
+	//
+	// The caller must not modify the content of the slice.
+	PublishResources(ctx context.Context, nodeResources []*resourceapi.ResourceModel)
+
 	// This unexported method ensures that we can modify the interface
 	// without causing an API break of the package
 	// (https://pkg.go.dev/golang.org/x/exp/apidiff#section-readme).
@@ -57,14 +75,6 @@ func DriverName(driverName string) Option {
 	}
 }
 
-// Logger overrides the default klog.Background logger.
-func Logger(logger klog.Logger) Option {
-	return func(o *options) error {
-		o.logger = logger
-		return nil
-	}
-}
-
 // GRPCVerbosity sets the verbosity for logging gRPC calls. Default is 4. A negative
 // value disables logging.
 func GRPCVerbosity(level int) Option {
@@ -162,15 +172,50 @@ func NodeV1alpha3(enabled bool) Option {
 	}
 }
 
+// KubeClient grants the plugin access to the API server. This is needed
+// for syncing ResourceSlice objects. It's the responsibility of the DRA driver
+// developer to ensure that this client has permission to read, write,
+// patch and list such objects. It also needs permission to read node objects.
+// Ideally, a validating admission policy should be used to limit write
+// access to ResourceSlices which belong to the node.
+func KubeClient(kubeClient kubernetes.Interface) Option {
+	return func(o *options) error {
+		o.kubeClient = kubeClient
+		return nil
+	}
+}
+
+// NodeName tells the plugin on which node it is running. This is needed for
+// syncing ResourceSlice objects.
+func NodeName(nodeName string) Option {
+	return func(o *options) error {
+		o.nodeName = nodeName
+		return nil
+	}
+}
+
+// NodeUID tells the plugin the UID of the v1.Node object. This is used
+// when syncing ResourceSlice objects, but doesn't have to be used. If
+// not supplied, the controller will look up the object once.
+func NodeUID(nodeUID types.UID) Option {
+	return func(o *options) error {
+		o.nodeUID = nodeUID
+		return nil
+	}
+}
+
 type options struct {
 	logger                     klog.Logger
 	grpcVerbosity              int
 	driverName                 string
+	nodeName                   string
+	nodeUID                    types.UID
 	draEndpoint                endpoint
 	draAddress                 string
 	pluginRegistrationEndpoint endpoint
 	unaryInterceptors          []grpc.UnaryServerInterceptor
 	streamInterceptors         []grpc.StreamServerInterceptor
+	kubeClient                 kubernetes.Interface
 
 	nodeV1alpha3 bool
 }
@@ -178,18 +223,36 @@ type options struct {
 // draPlugin combines the kubelet registration service and the DRA node plugin
 // service.
 type draPlugin struct {
-	registrar *nodeRegistrar
+	// backgroundCtx is for activities that are started later.
-	plugin    *grpcServer
+	backgroundCtx context.Context
+	// cancel cancels the backgroundCtx.
+	cancel     func(cause error)
+	wg         sync.WaitGroup
+	registrar  *nodeRegistrar
+	plugin     *grpcServer
+	driverName string
+	nodeName   string
+	nodeUID    types.UID
+	kubeClient kubernetes.Interface
+
+	// Information about resource publishing changes concurrently and thus
+	// must be protected by the mutex. The controller gets started only
+	// if needed.
+	mutex                   sync.Mutex
+	resourceSliceController *resourceslice.Controller
 }
 
 // Start sets up two gRPC servers (one for registration, one for the DRA node
 // client). By default, all APIs implemented by the nodeServer get registered.
-func Start(nodeServer interface{}, opts ...Option) (result DRAPlugin, finalErr error) {
+//
-	d := &draPlugin{}
+// The context and/or DRAPlugin.Stop can be used to stop all background activity.
-
+// Stop also blocks. A logger can be stored in the context to add values or
+// a name to all log entries.
+func Start(ctx context.Context, nodeServer interface{}, opts ...Option) (result DRAPlugin, finalErr error) {
+	logger := klog.FromContext(ctx)
 	o := options{
 		logger:        klog.Background(),
-		grpcVerbosity: 4,
+		grpcVerbosity: 6, // Logs requests and responses, which can be large.
 		nodeV1alpha3:  true,
 	}
 	for _, option := range opts {
@@ -212,12 +275,42 @@ func Start(nodeServer interface{}, opts ...Option) (result DRAPlugin, finalErr e
 		return nil, errors.New("a Unix domain socket path and/or listener must be set for the registrar")
 	}
 
+	d := &draPlugin{
+		driverName: o.driverName,
+		nodeName:   o.nodeName,
+		nodeUID:    o.nodeUID,
+		kubeClient: o.kubeClient,
+	}
+
+	// Stop calls cancel and therefore both cancellation
+	// and Stop cause goroutines to stop.
+	ctx, cancel := context.WithCancelCause(ctx)
+	d.backgroundCtx, d.cancel = ctx, cancel
+	logger.V(3).Info("Starting")
+	d.wg.Add(1)
+	go func() {
+		defer d.wg.Done()
+		defer logger.V(3).Info("Stopping")
+		<-ctx.Done()
+	}()
+
+	// Clean up if we don't finish succcessfully.
+	defer func() {
+		if r := recover(); r != nil {
+			d.Stop()
+			panic(r)
+		}
+		if finalErr != nil {
+			d.Stop()
+		}
+	}()
+
 	// Run the node plugin gRPC server first to ensure that it is ready.
 	implemented := false
-	plugin, err := startGRPCServer(klog.LoggerWithName(o.logger, "dra"), o.grpcVerbosity, o.unaryInterceptors, o.streamInterceptors, o.draEndpoint, func(grpcServer *grpc.Server) {
+	plugin, err := startGRPCServer(klog.NewContext(ctx, klog.LoggerWithName(logger, "dra")), o.grpcVerbosity, o.unaryInterceptors, o.streamInterceptors, o.draEndpoint, func(grpcServer *grpc.Server) {
-		if nodeServer, ok := nodeServer.(drapbv1alpha3.NodeServer); ok && o.nodeV1alpha3 {
+		if nodeServer, ok := nodeServer.(drapb.NodeServer); ok && o.nodeV1alpha3 {
-			o.logger.V(5).Info("registering drapbv1alpha3.NodeServer")
+			logger.V(5).Info("registering drapbv1alpha3.NodeServer")
-			drapbv1alpha3.RegisterNodeServer(grpcServer, nodeServer)
+			drapb.RegisterNodeServer(grpcServer, nodeServer)
 			implemented = true
 		}
 	})
@@ -225,38 +318,77 @@ func Start(nodeServer interface{}, opts ...Option) (result DRAPlugin, finalErr e
 		return nil, fmt.Errorf("start node client: %v", err)
 	}
 	d.plugin = plugin
-	defer func() {
-		// Clean up if we didn't finish succcessfully.
-		if r := recover(); r != nil {
-			plugin.stop()
-			panic(r)
-		}
-		if finalErr != nil {
-			plugin.stop()
-		}
-	}()
 	if !implemented {
 		return nil, errors.New("no supported DRA gRPC API is implemented and enabled")
 	}
 
 	// Now make it available to kubelet.
-	registrar, err := startRegistrar(klog.LoggerWithName(o.logger, "registrar"), o.grpcVerbosity, o.unaryInterceptors, o.streamInterceptors, o.driverName, o.draAddress, o.pluginRegistrationEndpoint)
+	registrar, err := startRegistrar(klog.NewContext(ctx, klog.LoggerWithName(logger, "registrar")), o.grpcVerbosity, o.unaryInterceptors, o.streamInterceptors, o.driverName, o.draAddress, o.pluginRegistrationEndpoint)
 	if err != nil {
 		return nil, fmt.Errorf("start registrar: %v", err)
 	}
 	d.registrar = registrar
 
+	// startGRPCServer and startRegistrar don't implement cancellation
+	// themselves, we add that for both here.
+	d.wg.Add(1)
+	go func() {
+		defer d.wg.Done()
+		<-ctx.Done()
+
+		// Time to stop.
+		d.plugin.stop()
+		d.registrar.stop()
+
+		// d.resourceSliceController is set concurrently.
+		d.mutex.Lock()
+		d.resourceSliceController.Stop()
+		d.mutex.Unlock()
+	}()
+
 	return d, nil
 }
 
+// Stop implements [DRAPlugin.Stop].
 func (d *draPlugin) Stop() {
 	if d == nil {
 		return
 	}
-	d.registrar.stop()
+	d.cancel(errors.New("DRA plugin was stopped"))
-	d.plugin.stop()
+	// Wait for goroutines in Start to clean up and exit.
+	d.wg.Wait()
 }
 
+// PublishResources implements [DRAPlugin.PublishResources].
+func (d *draPlugin) PublishResources(ctx context.Context, nodeResources []*resourceapi.ResourceModel) {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	owner := resourceslice.Owner{
+		APIVersion: "v1",
+		Kind:       "Node",
+		Name:       d.nodeName,
+		UID:        d.nodeUID, // Optional, will be determined by controller if empty.
+	}
+	resources := &resourceslice.Resources{NodeResources: nodeResources}
+	if d.resourceSliceController == nil {
+		// Start publishing the information. The controller is using
+		// our background context, not the one passed into this
+		// function, and thus is connected to the lifecycle of the
+		// plugin.
+		controllerCtx := d.backgroundCtx
+		controllerLogger := klog.FromContext(controllerCtx)
+		controllerLogger = klog.LoggerWithName(controllerLogger, "ResourceSlice controller")
+		controllerCtx = klog.NewContext(controllerCtx, controllerLogger)
+		d.resourceSliceController = resourceslice.StartController(controllerCtx, d.kubeClient, d.driverName, owner, resources)
+		return
+	}
+
+	// Inform running controller about new information.
+	d.resourceSliceController.Update(resources)
+}
+
+// RegistrationStatus implements [DRAPlugin.RegistrationStatus].
 func (d *draPlugin) RegistrationStatus() *registerapi.RegistrationStatus {
 	if d.registrar == nil {
 		return nil

staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/noderegistrar.go

@@ -17,30 +17,30 @@ limitations under the License.
 package kubeletplugin
 
 import (
+	"context"
 	"fmt"
 
 	"google.golang.org/grpc"
-	"k8s.io/klog/v2"
 	registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1"
 )
 
 type nodeRegistrar struct {
-	logger klog.Logger
 	registrationServer
 	server *grpcServer
 }
 
 // startRegistrar returns a running instance.
-func startRegistrar(logger klog.Logger, grpcVerbosity int, interceptors []grpc.UnaryServerInterceptor, streamInterceptors []grpc.StreamServerInterceptor, driverName string, endpoint string, pluginRegistrationEndpoint endpoint) (*nodeRegistrar, error) {
+//
+// The context is only used for additional values, cancellation is ignored.
+func startRegistrar(valueCtx context.Context, grpcVerbosity int, interceptors []grpc.UnaryServerInterceptor, streamInterceptors []grpc.StreamServerInterceptor, driverName string, endpoint string, pluginRegistrationEndpoint endpoint) (*nodeRegistrar, error) {
 	n := &nodeRegistrar{
-		logger: logger,
 		registrationServer: registrationServer{
 			driverName:        driverName,
 			endpoint:          endpoint,
 			supportedVersions: []string{"1.0.0"}, // TODO: is this correct?
 		},
 	}
-	s, err := startGRPCServer(logger, grpcVerbosity, interceptors, streamInterceptors, pluginRegistrationEndpoint, func(grpcServer *grpc.Server) {
+	s, err := startGRPCServer(valueCtx, grpcVerbosity, interceptors, streamInterceptors, pluginRegistrationEndpoint, func(grpcServer *grpc.Server) {
 		registerapi.RegisterRegistrationServer(grpcServer, n)
 	})
 	if err != nil {

staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/nonblockinggrpcserver.go

@@ -26,15 +26,17 @@ import (
 
 	"google.golang.org/grpc"
 	"k8s.io/klog/v2"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 )
 
+var requestID int64
+
 type grpcServer struct {
-	logger        klog.Logger
 	grpcVerbosity int
 	wg            sync.WaitGroup
 	endpoint      endpoint
 	server        *grpc.Server
-	requestID     int64
 }
 
 type registerService func(s *grpc.Server)
@@ -54,9 +56,11 @@ type endpoint struct {
 
 // startGRPCServer sets up the GRPC server on a Unix domain socket and spawns a goroutine
 // which handles requests for arbitrary services.
-func startGRPCServer(logger klog.Logger, grpcVerbosity int, unaryInterceptors []grpc.UnaryServerInterceptor, streamInterceptors []grpc.StreamServerInterceptor, endpoint endpoint, services ...registerService) (*grpcServer, error) {
+//
+// The context is only used for additional values, cancellation is ignored.
+func startGRPCServer(valueCtx context.Context, grpcVerbosity int, unaryInterceptors []grpc.UnaryServerInterceptor, streamInterceptors []grpc.StreamServerInterceptor, endpoint endpoint, services ...registerService) (*grpcServer, error) {
+	logger := klog.FromContext(valueCtx)
 	s := &grpcServer{
-		logger:        logger,
 		endpoint:      endpoint,
 		grpcVerbosity: grpcVerbosity,
 	}
@@ -79,10 +83,11 @@ func startGRPCServer(logger klog.Logger, grpcVerbosity int, unaryInterceptors []
 	// Run a gRPC server. It will close the listening socket when
 	// shutting down, so we don't need to do that.
 	var opts []grpc.ServerOption
-	var finalUnaryInterceptors []grpc.UnaryServerInterceptor
+	finalUnaryInterceptors := []grpc.UnaryServerInterceptor{unaryContextInterceptor(valueCtx)}
-	var finalStreamInterceptors []grpc.StreamServerInterceptor
+	finalStreamInterceptors := []grpc.StreamServerInterceptor{streamContextInterceptor(valueCtx)}
 	if grpcVerbosity >= 0 {
 		finalUnaryInterceptors = append(finalUnaryInterceptors, s.interceptor)
+		finalStreamInterceptors = append(finalStreamInterceptors, s.streamInterceptor)
 	}
 	finalUnaryInterceptors = append(finalUnaryInterceptors, unaryInterceptors...)
 	finalStreamInterceptors = append(finalStreamInterceptors, streamInterceptors...)
@@ -103,16 +108,66 @@ func startGRPCServer(logger klog.Logger, grpcVerbosity int, unaryInterceptors []
 		}
 	}()
 
-	logger.Info("GRPC server started")
+	logger.V(3).Info("GRPC server started")
 	return s, nil
 }
 
+// unaryContextInterceptor injects values from the context into the context
+// used by the call chain.
+func unaryContextInterceptor(valueCtx context.Context) grpc.UnaryServerInterceptor {
+	return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		ctx = mergeContexts(ctx, valueCtx)
+		return handler(ctx, req)
+	}
+}
+
+// streamContextInterceptor does the same as UnaryContextInterceptor for streams.
+func streamContextInterceptor(valueCtx context.Context) grpc.StreamServerInterceptor {
+	return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		ctx := mergeContexts(ss.Context(), valueCtx)
+		return handler(srv, mergeServerStream{ServerStream: ss, ctx: ctx})
+	}
+}
+
+type mergeServerStream struct {
+	grpc.ServerStream
+	ctx context.Context
+}
+
+func (m mergeServerStream) Context() context.Context {
+	return m.ctx
+}
+
+// mergeContexts creates a new context where cancellation is handled by the
+// root context. The values stored by the value context are used as fallback if
+// the root context doesn't have a certain value.
+func mergeContexts(rootCtx, valueCtx context.Context) context.Context {
+	return mergeCtx{
+		Context:  rootCtx,
+		valueCtx: valueCtx,
+	}
+}
+
+type mergeCtx struct {
+	context.Context
+	valueCtx context.Context
+}
+
+func (m mergeCtx) Value(i interface{}) interface{} {
+	if v := m.Context.Value(i); v != nil {
+		return v
+	}
+	return m.valueCtx.Value(i)
+}
+
 // interceptor is called for each request. It creates a logger with a unique,
 // sequentially increasing request ID and adds that logger to the context. It
 // also logs request and response.
 func (s *grpcServer) interceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
-	requestID := atomic.AddInt64(&s.requestID, 1)
+
-	logger := klog.LoggerWithValues(s.logger, "requestID", requestID)
+	requestID := atomic.AddInt64(&requestID, 1)
+	logger := klog.FromContext(ctx)
+	logger = klog.LoggerWithValues(logger, "requestID", requestID, "method", info.FullMethod)
 	ctx = klog.NewContext(ctx, logger)
 	logger.V(s.grpcVerbosity).Info("handling request", "request", req)
 	defer func() {
@@ -123,13 +178,57 @@ func (s *grpcServer) interceptor(ctx context.Context, req interface{}, info *grp
 	}()
 	resp, err = handler(ctx, req)
 	if err != nil {
-		logger.Error(err, "handling request failed", "request", req)
+		logger.Error(err, "handling request failed")
 	} else {
 		logger.V(s.grpcVerbosity).Info("handling request succeeded", "response", resp)
 	}
 	return
 }
 
+func (s *grpcServer) streamInterceptor(server interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	requestID := atomic.AddInt64(&requestID, 1)
+	ctx := stream.Context()
+	logger := klog.FromContext(ctx)
+	logger = klog.LoggerWithValues(logger, "requestID", requestID, "method", info.FullMethod)
+	ctx = klog.NewContext(ctx, logger)
+	stream = logStream{
+		ServerStream:  stream,
+		ctx:           ctx,
+		grpcVerbosity: s.grpcVerbosity,
+	}
+	logger.V(s.grpcVerbosity).Info("handling stream")
+	err := handler(server, stream)
+	if err != nil {
+		logger.Error(err, "handling stream failed")
+	} else {
+		logger.V(s.grpcVerbosity).Info("handling stream succeeded")
+	}
+	return err
+
+}
+
+type logStream struct {
+	grpc.ServerStream
+	ctx           context.Context
+	grpcVerbosity int
+}
+
+func (l logStream) Context() context.Context {
+	return l.ctx
+}
+
+func (l logStream) SendMsg(msg interface{}) error {
+	logger := klog.FromContext(l.ctx)
+	logger.V(l.grpcVerbosity).Info("sending stream message", "message", msg)
+	err := l.ServerStream.SendMsg(msg)
+	if err != nil {
+		logger.Error(err, "sending stream message failed")
+	} else {
+		logger.V(l.grpcVerbosity).Info("sending stream message succeeded")
+	}
+	return err
+}
+
 // stop ensures that the server is not running anymore and cleans up all resources.
 // It is idempotent and may be called with a nil pointer.
 func (s *grpcServer) stop() {
@@ -143,8 +242,7 @@ func (s *grpcServer) stop() {
 	s.server = nil
 	if s.endpoint.path != "" {
 		if err := os.Remove(s.endpoint.path); err != nil && !os.IsNotExist(err) {
-			s.logger.Error(err, "remove Unix socket")
+			utilruntime.HandleError(fmt.Errorf("remove Unix socket: %w", err))
 		}
 	}
-	s.logger.V(3).Info("GRPC server stopped")
 }

staging/src/k8s.io/dynamic-resource-allocation/resourceslice/noderesources.go

@@ -0,0 +1,391 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resourceslice
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+
+	resourceapi "k8s.io/api/resource/v1alpha2"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	resourceinformers "k8s.io/client-go/informers/resource/v1alpha2"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
+)
+
+const (
+	// resyncPeriod for informer
+	// TODO (https://github.com/kubernetes/kubernetes/issues/123688): disable?
+	resyncPeriod = time.Duration(10 * time.Minute)
+)
+
+// Controller synchronizes information about resources of one
+// driver with ResourceSlice objects. It currently supports node-local
+// resources. A DRA driver for node-local resources typically runs this
+// controller as part of its kubelet plugin.
+//
+// Support for network-attached resources will be added later.
+type Controller struct {
+	cancel     func(cause error)
+	driverName string
+	owner      Owner
+	kubeClient kubernetes.Interface
+	wg         sync.WaitGroup
+	queue      workqueue.TypedRateLimitingInterface[string]
+	sliceStore cache.Store
+
+	mutex sync.RWMutex
+
+	// When receiving updates from the driver, the entire pointer replaced,
+	// so it is okay to not do a deep copy of it when reading it. Only reading
+	// the pointer itself must be protected by a read lock.
+	resources *Resources
+}
+
+// Resources is a complete description of all resources synchronized by the controller.
+type Resources struct {
+	// NodeResources are resources that are local to one node.
+	NodeResources []*resourceapi.ResourceModel
+}
+
+type Owner struct {
+	APIVersion string
+	Kind       string
+	Name       string
+	UID        types.UID
+}
+
+// StartController constructs a new controller and starts it.
+// If the owner is a v1.Node, then the NodeName field in the
+// ResourceSlice objects is set and used to identify objects
+// managed by the controller. The UID is not needed in that
+// case, the controller will determine it automatically.
+//
+// If a kubeClient is provided, then it synchronizes ResourceSlices
+// with the resource information provided by plugins. Without it,
+// the controller is inactive. This can happen when kubelet is run stand-alone
+// without an apiserver. In that case we can't and don't need to publish
+// ResourceSlices.
+func StartController(ctx context.Context, kubeClient kubernetes.Interface, driverName string, owner Owner, resources *Resources) *Controller {
+	if kubeClient == nil {
+		return nil
+	}
+
+	logger := klog.FromContext(ctx)
+	ctx, cancel := context.WithCancelCause(ctx)
+
+	c := &Controller{
+		cancel:     cancel,
+		kubeClient: kubeClient,
+		driverName: driverName,
+		owner:      owner,
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: "node_resource_slices"},
+		),
+		resources: resources,
+	}
+
+	logger.V(3).Info("Starting")
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		defer logger.V(3).Info("Stopping")
+		c.run(ctx)
+	}()
+
+	// Sync once.
+	c.queue.Add("")
+
+	return c
+}
+
+// Stop cancels all background activity and blocks until the controller has stopped.
+func (c *Controller) Stop() {
+	if c == nil {
+		return
+	}
+	c.cancel(errors.New("ResourceSlice controller was asked to stop"))
+	c.wg.Wait()
+}
+
+// Update sets the new desired state of the resource information.
+func (c *Controller) Update(resources *Resources) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	c.resources = resources
+	c.queue.Add("")
+}
+
+// run is running in the background. It handles blocking initialization (like
+// syncing the informer) and then syncs the actual with the desired state.
+func (c *Controller) run(ctx context.Context) {
+	logger := klog.FromContext(ctx)
+
+	// We always filter by driver name, by node name only for node-local resources.
+	selector := fields.Set{"driverName": c.driverName}
+	if c.owner.APIVersion == "v1" && c.owner.Kind == "Node" {
+		selector["nodeName"] = c.owner.Name
+	}
+	informer := resourceinformers.NewFilteredResourceSliceInformer(c.kubeClient, resyncPeriod, nil, func(options *metav1.ListOptions) {
+		options.FieldSelector = selector.String()
+	})
+	c.sliceStore = informer.GetStore()
+	handler, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj any) {
+			slice, ok := obj.(*resourceapi.ResourceSlice)
+			if !ok {
+				return
+			}
+			logger.V(5).Info("ResourceSlice add", "slice", klog.KObj(slice))
+			c.queue.Add("")
+		},
+		UpdateFunc: func(old, new any) {
+			oldSlice, ok := old.(*resourceapi.ResourceSlice)
+			if !ok {
+				return
+			}
+			newSlice, ok := new.(*resourceapi.ResourceSlice)
+			if !ok {
+				return
+			}
+			if loggerV := logger.V(6); loggerV.Enabled() {
+				loggerV.Info("ResourceSlice update", "slice", klog.KObj(newSlice), "diff", cmp.Diff(oldSlice, newSlice))
+			} else {
+				logger.V(5).Info("ResourceSlice update", "slice", klog.KObj(newSlice))
+			}
+			c.queue.Add("")
+		},
+		DeleteFunc: func(obj any) {
+			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
+				obj = tombstone.Obj
+			}
+			slice, ok := obj.(*resourceapi.ResourceSlice)
+			if !ok {
+				return
+			}
+			logger.V(5).Info("ResourceSlice delete", "slice", klog.KObj(slice))
+			c.queue.Add("")
+		},
+	})
+	if err != nil {
+		logger.Error(err, "Registering event handler on the ResourceSlice informer failed, disabling resource monitoring")
+		return
+	}
+
+	// Start informer and wait for our cache to be populated.
+	logger.V(3).Info("Starting ResourceSlice informer and waiting for it to sync")
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		defer logger.V(3).Info("ResourceSlice informer has stopped")
+		defer c.queue.ShutDown() // Once we get here, we must have been asked to stop.
+		informer.Run(ctx.Done())
+	}()
+	for !handler.HasSynced() {
+		select {
+		case <-time.After(time.Second):
+		case <-ctx.Done():
+			return
+		}
+	}
+	logger.V(3).Info("ResourceSlice informer has synced")
+
+	for c.processNextWorkItem(ctx) {
+	}
+}
+
+func (c *Controller) processNextWorkItem(ctx context.Context) bool {
+	key, shutdown := c.queue.Get()
+	if shutdown {
+		return false
+	}
+	defer c.queue.Done(key)
+
+	// Panics are caught and treated like errors.
+	var err error
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				err = fmt.Errorf("internal error: %v", r)
+			}
+		}()
+		err = c.sync(ctx)
+	}()
+
+	if err != nil {
+		utilruntime.HandleErrorWithContext(ctx, err, "processing ResourceSlice objects")
+		c.queue.AddRateLimited(key)
+
+		// Return without removing the work item from the queue.
+		// It will be retried.
+		return true
+	}
+
+	c.queue.Forget(key)
+	return true
+}
+
+func (c *Controller) sync(ctx context.Context) error {
+	logger := klog.FromContext(ctx)
+
+	// Gather information about the actual and desired state.
+	slices := c.sliceStore.List()
+	var resources *Resources
+	c.mutex.RLock()
+	resources = c.resources
+	c.mutex.RUnlock()
+
+	// Resources that are not yet stored in any slice need to be published.
+	// Here we track the indices of any resources that are already stored.
+	storedResourceIndices := sets.New[int]()
+
+	// Slices that don't match any driver resource can either be updated (if there
+	// are new driver resources that need to be stored) or they need to be deleted.
+	obsoleteSlices := make([]*resourceapi.ResourceSlice, 0, len(slices))
+
+	// Match slices with resource information.
+	for _, obj := range slices {
+		slice := obj.(*resourceapi.ResourceSlice)
+
+		// TODO: network-attached resources.
+		index := indexOfModel(resources.NodeResources, &slice.ResourceModel)
+		if index >= 0 {
+			storedResourceIndices.Insert(index)
+			continue
+		}
+
+		obsoleteSlices = append(obsoleteSlices, slice)
+	}
+
+	if loggerV := logger.V(6); loggerV.Enabled() {
+		// Dump entire resource information.
+		loggerV.Info("Syncing existing driver resource slices with driver resources", "slices", klog.KObjSlice(slices), "resources", resources)
+	} else {
+		logger.V(5).Info("Syncing existing driver resource slices with driver resources", "slices", klog.KObjSlice(slices), "numResources", len(resources.NodeResources))
+	}
+
+	// Retrieve node object to get UID?
+	// The result gets cached and is expected to not change while
+	// the controller runs.
+	if c.owner.UID == "" && c.owner.APIVersion == "v1" && c.owner.Kind == "Node" {
+		node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, c.owner.Name, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("retrieve node %q: %w", c.owner.Name, err)
+		}
+		// There is only one worker, so no locking needed.
+		c.owner.UID = node.UID
+	}
+
+	// Update stale slices before removing what's left.
+	//
+	// We don't really know which of these slices might have
+	// been used for "the" driver resource because they don't
+	// have a unique ID. In practice, a driver is most likely
+	// to just give us one ResourceModel, in which case
+	// this isn't a problem at all. If we have more than one,
+	// then at least conceptually it currently doesn't matter
+	// where we publish it.
+	//
+	// The long-term goal is to move the handling of
+	// ResourceSlice objects into the driver, with kubelet
+	// just acting as a REST proxy. The advantage of that will
+	// be that kubelet won't need to support the same
+	// resource API version as the driver and the control plane.
+	// With that approach, the driver will be able to match
+	// up objects more intelligently.
+	numObsoleteSlices := len(obsoleteSlices)
+	for index, resource := range resources.NodeResources {
+		if storedResourceIndices.Has(index) {
+			// No need to do anything, it is already stored exactly
+			// like this in an existing slice.
+			continue
+		}
+
+		if numObsoleteSlices > 0 {
+			// Update one existing slice.
+			slice := obsoleteSlices[numObsoleteSlices-1]
+			numObsoleteSlices--
+			slice = slice.DeepCopy()
+			slice.ResourceModel = *resource
+			logger.V(5).Info("Reusing existing resource slice", "slice", klog.KObj(slice))
+			if _, err := c.kubeClient.ResourceV1alpha2().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}); err != nil {
+				return fmt.Errorf("update resource slice: %w", err)
+			}
+			continue
+		}
+
+		// Create a new slice.
+		slice := &resourceapi.ResourceSlice{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: c.owner.Name + "-" + c.driverName + "-",
+				OwnerReferences: []metav1.OwnerReference{
+					{
+						APIVersion: c.owner.APIVersion,
+						Kind:       c.owner.Kind,
+						Name:       c.owner.Name,
+						UID:        c.owner.UID,
+						Controller: ptr.To(true),
+					},
+				},
+			},
+			DriverName:    c.driverName,
+			ResourceModel: *resource,
+		}
+		if c.owner.APIVersion == "v1" && c.owner.Kind == "Node" {
+			slice.NodeName = c.owner.Name
+		}
+		logger.V(5).Info("Creating new resource slice", "slice", klog.KObj(slice))
+		if _, err := c.kubeClient.ResourceV1alpha2().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}); err != nil {
+			return fmt.Errorf("create resource slice: %w", err)
+		}
+	}
+
+	// All remaining slices are truly orphaned.
+	for i := 0; i < numObsoleteSlices; i++ {
+		slice := obsoleteSlices[i]
+		logger.V(5).Info("Deleting obsolete resource slice", "slice", klog.KObj(slice))
+		if err := c.kubeClient.ResourceV1alpha2().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{}); err != nil {
+			return fmt.Errorf("delete resource slice: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func indexOfModel(models []*resourceapi.ResourceModel, model *resourceapi.ResourceModel) int {
+	for index, m := range models {
+		if apiequality.Semantic.DeepEqual(m, model) {
+			return index
+		}
+	}
+	return -1
+}

staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.pb.go

@@ -17,7 +17,7 @@ limitations under the License.
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
 // source: api.proto
 
-package v1alpha3
+package v1alpha4
 
 import (
 	context "context"
@@ -29,7 +29,6 @@ import (
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
 	io "io"
-	v1alpha2 "k8s.io/api/resource/v1alpha2"
 	math "math"
 	math_bits "math/bits"
 	reflect "reflect"
@@ -342,88 +341,6 @@ func (m *NodeUnprepareResourceResponse) GetError() string {
 	return ""
 }
 
-type NodeListAndWatchResourcesRequest struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *NodeListAndWatchResourcesRequest) Reset()      { *m = NodeListAndWatchResourcesRequest{} }
-func (*NodeListAndWatchResourcesRequest) ProtoMessage() {}
-func (*NodeListAndWatchResourcesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_00212fb1f9d3bf1c, []int{6}
-}
-func (m *NodeListAndWatchResourcesRequest) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *NodeListAndWatchResourcesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_NodeListAndWatchResourcesRequest.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *NodeListAndWatchResourcesRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_NodeListAndWatchResourcesRequest.Merge(m, src)
-}
-func (m *NodeListAndWatchResourcesRequest) XXX_Size() int {
-	return m.Size()
-}
-func (m *NodeListAndWatchResourcesRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_NodeListAndWatchResourcesRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_NodeListAndWatchResourcesRequest proto.InternalMessageInfo
-
-type NodeListAndWatchResourcesResponse struct {
-	Resources            []*v1alpha2.ResourceModel `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}                  `json:"-"`
-	XXX_sizecache        int32                     `json:"-"`
-}
-
-func (m *NodeListAndWatchResourcesResponse) Reset()      { *m = NodeListAndWatchResourcesResponse{} }
-func (*NodeListAndWatchResourcesResponse) ProtoMessage() {}
-func (*NodeListAndWatchResourcesResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_00212fb1f9d3bf1c, []int{7}
-}
-func (m *NodeListAndWatchResourcesResponse) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *NodeListAndWatchResourcesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_NodeListAndWatchResourcesResponse.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *NodeListAndWatchResourcesResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_NodeListAndWatchResourcesResponse.Merge(m, src)
-}
-func (m *NodeListAndWatchResourcesResponse) XXX_Size() int {
-	return m.Size()
-}
-func (m *NodeListAndWatchResourcesResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_NodeListAndWatchResourcesResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_NodeListAndWatchResourcesResponse proto.InternalMessageInfo
-
-func (m *NodeListAndWatchResourcesResponse) GetResources() []*v1alpha2.ResourceModel {
-	if m != nil {
-		return m.Resources
-	}
-	return nil
-}
-
 type Claim struct {
 	// The ResourceClaim namespace (ResourceClaim.meta.Namespace).
 	// This field is REQUIRED.
@@ -433,24 +350,15 @@ type Claim struct {
 	Uid string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"`
 	// The name of the Resource claim (ResourceClaim.meta.Name)
 	// This field is REQUIRED.
-	Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
+	Name                 string   `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
-	// Resource handle (AllocationResult.ResourceHandles[*].Data)
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	// This field is REQUIRED.
+	XXX_sizecache        int32    `json:"-"`
-	ResourceHandle string `protobuf:"bytes,4,opt,name=resource_handle,json=resourceHandle,proto3" json:"resource_handle,omitempty"`
-	// Structured parameter resource handle (AllocationResult.ResourceHandles[*].StructuredData).
-	// This field is OPTIONAL. If present, it needs to be used
-	// instead of resource_handle. It will only have a single entry.
-	//
-	// Using "repeated" instead of "optional" is a workaround for https://github.com/gogo/protobuf/issues/713.
-	StructuredResourceHandle []*v1alpha2.StructuredResourceHandle `protobuf:"bytes,5,rep,name=structured_resource_handle,json=structuredResourceHandle,proto3" json:"structured_resource_handle,omitempty"`
-	XXX_NoUnkeyedLiteral     struct{}                             `json:"-"`
-	XXX_sizecache            int32                                `json:"-"`
 }
 
 func (m *Claim) Reset()      { *m = Claim{} }
 func (*Claim) ProtoMessage() {}
 func (*Claim) Descriptor() ([]byte, []int) {
-	return fileDescriptor_00212fb1f9d3bf1c, []int{8}
+	return fileDescriptor_00212fb1f9d3bf1c, []int{6}
 }
 func (m *Claim) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -500,20 +408,6 @@ func (m *Claim) GetName() string {
 	return ""
 }
 
-func (m *Claim) GetResourceHandle() string {
-	if m != nil {
-		return m.ResourceHandle
-	}
-	return ""
-}
-
-func (m *Claim) GetStructuredResourceHandle() []*v1alpha2.StructuredResourceHandle {
-	if m != nil {
-		return m.StructuredResourceHandle
-	}
-	return nil
-}
-
 func init() {
 	proto.RegisterType((*NodePrepareResourcesRequest)(nil), "v1alpha3.NodePrepareResourcesRequest")
 	proto.RegisterType((*NodePrepareResourcesResponse)(nil), "v1alpha3.NodePrepareResourcesResponse")
@@ -523,55 +417,44 @@ func init() {
 	proto.RegisterType((*NodeUnprepareResourcesResponse)(nil), "v1alpha3.NodeUnprepareResourcesResponse")
 	proto.RegisterMapType((map[string]*NodeUnprepareResourceResponse)(nil), "v1alpha3.NodeUnprepareResourcesResponse.ClaimsEntry")
 	proto.RegisterType((*NodeUnprepareResourceResponse)(nil), "v1alpha3.NodeUnprepareResourceResponse")
-	proto.RegisterType((*NodeListAndWatchResourcesRequest)(nil), "v1alpha3.NodeListAndWatchResourcesRequest")
-	proto.RegisterType((*NodeListAndWatchResourcesResponse)(nil), "v1alpha3.NodeListAndWatchResourcesResponse")
 	proto.RegisterType((*Claim)(nil), "v1alpha3.Claim")
 }
 
 func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) }
 
 var fileDescriptor_00212fb1f9d3bf1c = []byte{
-	// 631 bytes of a gzipped FileDescriptorProto
+	// 481 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xcf, 0x6e, 0xd3, 0x4c,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0x4d, 0x6f, 0xd3, 0x40,
-	0x10, 0xcf, 0xb6, 0x4d, 0xf5, 0x65, 0x22, 0xb5, 0x9f, 0x56, 0x15, 0x0a, 0xa6, 0x98, 0x60, 0x51,
+	0x10, 0xcd, 0x36, 0x4d, 0x85, 0x27, 0x12, 0xa0, 0x55, 0x85, 0xa2, 0x50, 0x4c, 0x64, 0x51, 0x92,
-	0x12, 0x51, 0xb0, 0xc1, 0x05, 0x54, 0x81, 0x38, 0xd0, 0x16, 0xd4, 0xa2, 0x82, 0x90, 0x11, 0x42,
+	0x0b, 0xb6, 0x48, 0x8b, 0x54, 0x81, 0xb8, 0xa4, 0x05, 0xf1, 0x25, 0x84, 0x2c, 0x71, 0xe1, 0x02,
-	0xe2, 0x52, 0x36, 0xde, 0xc5, 0xb1, 0xe2, 0xd8, 0x66, 0xd7, 0xae, 0xe8, 0x8d, 0x47, 0xe0, 0xb1,
+	0x6b, 0x7b, 0x70, 0x57, 0xf9, 0xd8, 0x65, 0xd7, 0x8e, 0xd4, 0x1b, 0x3f, 0x81, 0x9f, 0xd5, 0x03,
-	0x7a, 0xe0, 0x80, 0x38, 0xf5, 0x84, 0x68, 0xb8, 0xf1, 0x14, 0xc8, 0x6b, 0x6f, 0xda, 0x44, 0x4e,
+	0x07, 0xc4, 0x89, 0x53, 0x45, 0xcd, 0x1f, 0x41, 0x5e, 0x3b, 0xe9, 0x87, 0x9c, 0x26, 0x52, 0x6f,
-	0x52, 0x89, 0xdb, 0xec, 0xfc, 0xf9, 0xcd, 0xce, 0x6f, 0x66, 0x76, 0xa1, 0x46, 0x62, 0xdf, 0x8c,
+	0x33, 0xe3, 0x9d, 0x79, 0x6f, 0xde, 0x1b, 0x19, 0x2c, 0x26, 0xb9, 0x2b, 0x95, 0x48, 0x04, 0xbd,
-	0x79, 0x94, 0x44, 0xf8, 0xbf, 0xc3, 0x7b, 0x24, 0x88, 0xbb, 0x64, 0x43, 0xbb, 0xe3, 0xf9, 0x49,
+	0x31, 0x7d, 0xcc, 0x46, 0xf2, 0x90, 0xed, 0xb4, 0x1f, 0xc5, 0x3c, 0x39, 0x4c, 0x03, 0x37, 0x14,
-	0x37, 0xed, 0x98, 0x6e, 0xd4, 0xb7, 0xbc, 0xc8, 0x8b, 0x2c, 0xe9, 0xd0, 0x49, 0x3f, 0xca, 0x93,
+	0x63, 0x2f, 0x16, 0xb1, 0xf0, 0xcc, 0x83, 0x20, 0xfd, 0x6a, 0x32, 0x93, 0x98, 0xa8, 0x68, 0x74,
-	0x3c, 0x48, 0x29, 0x0f, 0xd4, 0x6e, 0xf7, 0x36, 0x85, 0xe9, 0x47, 0x16, 0x89, 0x7d, 0x8b, 0x33,
+	0x5e, 0xc2, 0xdd, 0xf7, 0x22, 0xc2, 0x0f, 0x0a, 0x25, 0x53, 0xe8, 0xa3, 0x16, 0xa9, 0x0a, 0x51,
-	0x11, 0xa5, 0xdc, 0x65, 0x56, 0x01, 0x66, 0x5b, 0x1e, 0x0b, 0x19, 0x27, 0x09, 0xa3, 0xb9, 0xb7,
+	0xfb, 0xf8, 0x2d, 0x45, 0x9d, 0xd0, 0x2e, 0x6c, 0x84, 0x23, 0xc6, 0xc7, 0xba, 0x45, 0x3a, 0xf5,
-	0xf1, 0x1c, 0xae, 0xbc, 0x8a, 0x28, 0x7b, 0xcd, 0x59, 0x4c, 0x38, 0x73, 0x0a, 0x7f, 0xe1, 0xb0,
+	0x5e, 0xb3, 0x7f, 0xcb, 0x9d, 0x01, 0xb9, 0xfb, 0x79, 0xdd, 0x2f, 0x3f, 0x3b, 0x3f, 0x09, 0x6c,
-	0x4f, 0x29, 0x13, 0x09, 0x6e, 0xc1, 0xa2, 0x1b, 0x10, 0xbf, 0x2f, 0x1a, 0xa8, 0x39, 0xdf, 0xae,
+	0x55, 0x0f, 0xd2, 0x52, 0x4c, 0x34, 0xd2, 0x37, 0x97, 0x26, 0xf5, 0xcf, 0x26, 0x5d, 0xd5, 0x57,
-	0xdb, 0xcb, 0xa6, 0xba, 0x96, 0xb9, 0x9d, 0xe9, 0x9d, 0xc2, 0x6c, 0x7c, 0x43, 0xb0, 0x5a, 0x0e,
+	0xc0, 0xe8, 0x17, 0x93, 0x44, 0x1d, 0xcd, 0xc0, 0xda, 0x5f, 0xa0, 0x79, 0xae, 0x4c, 0x6f, 0x43,
-	0x24, 0xe2, 0x28, 0x14, 0x0c, 0xbf, 0x18, 0x43, 0xb2, 0xcf, 0x90, 0xa6, 0xc5, 0xe5, 0x69, 0xc4,
+	0x7d, 0x88, 0x47, 0x2d, 0xd2, 0x21, 0x3d, 0xcb, 0xcf, 0x43, 0xfa, 0x0c, 0x1a, 0x53, 0x36, 0x4a,
-	0xb3, 0x30, 0xe1, 0x47, 0x2a, 0x99, 0xf6, 0x01, 0xea, 0xe7, 0xd4, 0xf8, 0x7f, 0x98, 0xef, 0xb1,
+	0xb1, 0xb5, 0xd6, 0x21, 0xbd, 0x66, 0x7f, 0xfb, 0x4a, 0xac, 0x19, 0x94, 0x5f, 0xf4, 0x3c, 0x5d,
-	0xa3, 0x06, 0x6a, 0xa2, 0x76, 0xcd, 0xc9, 0x44, 0xfc, 0x18, 0xaa, 0x87, 0x24, 0x48, 0x59, 0x63,
+	0xdb, 0x23, 0x4e, 0x54, 0x29, 0xcb, 0x7c, 0x19, 0x0f, 0x9a, 0x61, 0xc4, 0x3f, 0x47, 0x38, 0xe5,
-	0xae, 0x89, 0xda, 0x75, 0x7b, 0x6d, 0x6a, 0x2e, 0x95, 0xca, 0xc9, 0x63, 0x1e, 0xcd, 0x6d, 0x22,
+	0x21, 0x16, 0x1b, 0x59, 0x83, 0x9b, 0xd9, 0xc9, 0x7d, 0xd8, 0x3f, 0x78, 0x7d, 0x50, 0x54, 0x7d,
-	0x83, 0x96, 0xd2, 0x32, 0x2c, 0xc6, 0x82, 0xba, 0x4b, 0xfd, 0x03, 0xca, 0x0e, 0x7d, 0x97, 0xe5,
+	0x08, 0x23, 0x5e, 0xc6, 0x74, 0x13, 0x1a, 0xa8, 0x94, 0x50, 0x86, 0x90, 0xe5, 0x17, 0x89, 0xf3,
-	0x15, 0xd5, 0xb6, 0x96, 0x06, 0x3f, 0xaf, 0xc1, 0xf6, 0xce, 0xde, 0x4e, 0xae, 0x75, 0xc0, 0xa5,
+	0x0a, 0xee, 0xe5, 0x28, 0x1f, 0x27, 0xf2, 0xba, 0xf2, 0xff, 0x26, 0x60, 0x2f, 0x1a, 0x55, 0x72,
-	0x7e, 0x21, 0xe3, 0x15, 0xa8, 0x32, 0xce, 0x23, 0x2e, 0x2f, 0x54, 0x73, 0xf2, 0x83, 0xb1, 0x0b,
+	0x7e, 0x77, 0x69, 0xd6, 0xee, 0x45, 0x51, 0x16, 0x77, 0x56, 0x5a, 0x10, 0x2c, 0xb3, 0xe0, 0xf9,
-	0x57, 0xb3, 0x2c, 0x6f, 0xc3, 0xf8, 0x5f, 0xe9, 0xff, 0x81, 0x40, 0x9f, 0x04, 0x55, 0xdc, 0x79,
+	0x45, 0x0b, 0xba, 0x4b, 0xd0, 0xaa, 0x4c, 0x78, 0xb2, 0x40, 0x9e, 0xf9, 0x4a, 0x73, 0x55, 0xc9,
-	0x7f, 0x0c, 0xeb, 0xfe, 0x28, 0x29, 0x93, 0x23, 0x4b, 0x5b, 0xd0, 0x99, 0xd5, 0x82, 0x27, 0xa3,
+	0x79, 0x55, 0xdf, 0x42, 0xc3, 0x50, 0xa3, 0x5b, 0x60, 0x4d, 0xd8, 0x18, 0xb5, 0x64, 0x21, 0x96,
-	0x2d, 0x68, 0xcd, 0xc8, 0x56, 0xd6, 0x84, 0x07, 0x13, 0xe8, 0x19, 0x96, 0x34, 0x64, 0x15, 0x9d,
+	0x4f, 0xce, 0x0a, 0x39, 0xe5, 0x94, 0x47, 0xa5, 0x21, 0x79, 0x48, 0x29, 0xac, 0xe7, 0x9f, 0x5b,
-	0x67, 0xd5, 0x80, 0x66, 0x16, 0xb6, 0xef, 0x8b, 0xe4, 0x69, 0x48, 0xdf, 0x91, 0xc4, 0xed, 0x8e,
+	0x75, 0x53, 0x32, 0x71, 0xff, 0x84, 0xc0, 0x7a, 0x4e, 0x82, 0xc6, 0xb0, 0x59, 0x75, 0xa7, 0x74,
-	0x13, 0x6b, 0x84, 0x70, 0x7d, 0x8a, 0x4f, 0x01, 0xbf, 0x07, 0x35, 0xb5, 0x40, 0x8a, 0xb4, 0x75,
+	0x7b, 0xd9, 0x1d, 0x1b, 0x27, 0xdb, 0x0f, 0x57, 0x3b, 0x77, 0xa7, 0x46, 0xc7, 0x70, 0xa7, 0xda,
-	0x33, 0xdf, 0x2e, 0x33, 0x5b, 0x54, 0x65, 0x54, 0xa5, 0xd9, 0xa6, 0xc2, 0x78, 0x19, 0x51, 0x16,
+	0x0f, 0xda, 0x5d, 0xee, 0x58, 0x01, 0xd6, 0x5b, 0xd5, 0x5a, 0xa7, 0x36, 0x18, 0x1c, 0x9f, 0xda,
-	0x38, 0x67, 0xd1, 0xc6, 0x1f, 0x04, 0x55, 0xc9, 0x17, 0x5e, 0x85, 0x5a, 0x48, 0xfa, 0x4c, 0xc4,
+	0xe4, 0xcf, 0xa9, 0x5d, 0xfb, 0x9e, 0xd9, 0xe4, 0x38, 0xb3, 0xc9, 0xaf, 0xcc, 0x26, 0x7f, 0x33,
-	0xc4, 0x65, 0xc5, 0xbd, 0xcf, 0x14, 0x19, 0x8f, 0xa9, 0x4f, 0x8b, 0x29, 0xc9, 0x44, 0x8c, 0x61,
+	0x9b, 0xfc, 0xf8, 0x67, 0xd7, 0x3e, 0x3d, 0x18, 0xee, 0x69, 0x97, 0x0b, 0x6f, 0x98, 0x06, 0x38,
-	0x21, 0x33, 0x37, 0xe6, 0xa5, 0x4a, 0xca, 0xb8, 0x05, 0xcb, 0x0a, 0xfa, 0xa0, 0x4b, 0x42, 0x1a,
+	0xc2, 0xc4, 0x93, 0xc3, 0xd8, 0x63, 0x92, 0x6b, 0x2f, 0x52, 0xcc, 0x2b, 0x41, 0x76, 0x83, 0x0d,
-	0xb0, 0xc6, 0x82, 0x34, 0x2f, 0x29, 0xf5, 0xae, 0xd4, 0xe2, 0x04, 0x34, 0x91, 0xf0, 0xd4, 0x4d,
+	0xf3, 0x2f, 0xd9, 0xf9, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x65, 0xc5, 0xc2, 0x0e, 0x91, 0x04, 0x00,
-	0x52, 0xce, 0xe8, 0xc1, 0x78, 0x4c, 0x55, 0x96, 0xf4, 0x70, 0x7a, 0x49, 0x6f, 0x86, 0xf1, 0xce,
+	0x00,
-	0x08, 0xb6, 0xd3, 0x10, 0x13, 0x2c, 0xf6, 0xc9, 0x1c, 0x2c, 0x64, 0xec, 0x62, 0x0f, 0x56, 0xca,
-	0x76, 0x1b, 0xaf, 0xcd, 0xda, 0x7d, 0xd9, 0x24, 0xed, 0xe6, 0xc5, 0x9e, 0x08, 0xa3, 0x82, 0xfb,
-	0x70, 0xa9, 0x7c, 0x86, 0x71, 0x6b, 0xf6, 0x94, 0xe7, 0xc9, 0xda, 0x17, 0x5d, 0x07, 0xa3, 0x82,
-	0x3f, 0xc3, 0xe5, 0x89, 0xd3, 0x83, 0x6f, 0x8d, 0x02, 0x4d, 0x1b, 0x43, 0x6d, 0xfd, 0x42, 0xbe,
-	0x2a, 0xef, 0x5d, 0xb4, 0xb5, 0x75, 0x7c, 0xaa, 0xa3, 0x93, 0x53, 0xbd, 0xf2, 0x65, 0xa0, 0xa3,
-	0xe3, 0x81, 0x8e, 0xbe, 0x0f, 0x74, 0xf4, 0x6b, 0xa0, 0xa3, 0xaf, 0xbf, 0xf5, 0xca, 0xfb, 0x1b,
-	0xc5, 0xd3, 0xdf, 0x4b, 0x3b, 0x2c, 0x60, 0x89, 0x15, 0xf7, 0xbc, 0xec, 0x1b, 0x10, 0x16, 0xe5,
-	0x44, 0x7d, 0x01, 0x1b, 0x9d, 0x45, 0xf9, 0xf2, 0x6f, 0xfc, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xb2,
-	0x0b, 0x57, 0x0c, 0x6d, 0x06, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -594,10 +477,6 @@ type NodeClient interface {
 	// NodeUnprepareResources is the opposite of NodePrepareResources.
 	// The same error handling rules apply,
 	NodeUnprepareResources(ctx context.Context, in *NodeUnprepareResourcesRequest, opts ...grpc.CallOption) (*NodeUnprepareResourcesResponse, error)
-	// NodeListAndWatchResources returns a stream of NodeResourcesResponse objects.
-	// At the start and whenever resource availability changes, the
-	// plugin must send one such object with all information to the Kubelet.
-	NodeListAndWatchResources(ctx context.Context, in *NodeListAndWatchResourcesRequest, opts ...grpc.CallOption) (Node_NodeListAndWatchResourcesClient, error)
 }
 
 type nodeClient struct {
@@ -626,38 +505,6 @@ func (c *nodeClient) NodeUnprepareResources(ctx context.Context, in *NodeUnprepa
 	return out, nil
 }
 
-func (c *nodeClient) NodeListAndWatchResources(ctx context.Context, in *NodeListAndWatchResourcesRequest, opts ...grpc.CallOption) (Node_NodeListAndWatchResourcesClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Node_serviceDesc.Streams[0], "/v1alpha3.Node/NodeListAndWatchResources", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &nodeNodeListAndWatchResourcesClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type Node_NodeListAndWatchResourcesClient interface {
-	Recv() (*NodeListAndWatchResourcesResponse, error)
-	grpc.ClientStream
-}
-
-type nodeNodeListAndWatchResourcesClient struct {
-	grpc.ClientStream
-}
-
-func (x *nodeNodeListAndWatchResourcesClient) Recv() (*NodeListAndWatchResourcesResponse, error) {
-	m := new(NodeListAndWatchResourcesResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
 // NodeServer is the server API for Node service.
 type NodeServer interface {
 	// NodePrepareResources prepares several ResourceClaims
@@ -668,10 +515,6 @@ type NodeServer interface {
 	// NodeUnprepareResources is the opposite of NodePrepareResources.
 	// The same error handling rules apply,
 	NodeUnprepareResources(context.Context, *NodeUnprepareResourcesRequest) (*NodeUnprepareResourcesResponse, error)
-	// NodeListAndWatchResources returns a stream of NodeResourcesResponse objects.
-	// At the start and whenever resource availability changes, the
-	// plugin must send one such object with all information to the Kubelet.
-	NodeListAndWatchResources(*NodeListAndWatchResourcesRequest, Node_NodeListAndWatchResourcesServer) error
 }
 
 // UnimplementedNodeServer can be embedded to have forward compatible implementations.
@@ -684,9 +527,6 @@ func (*UnimplementedNodeServer) NodePrepareResources(ctx context.Context, req *N
 func (*UnimplementedNodeServer) NodeUnprepareResources(ctx context.Context, req *NodeUnprepareResourcesRequest) (*NodeUnprepareResourcesResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method NodeUnprepareResources not implemented")
 }
-func (*UnimplementedNodeServer) NodeListAndWatchResources(req *NodeListAndWatchResourcesRequest, srv Node_NodeListAndWatchResourcesServer) error {
-	return status.Errorf(codes.Unimplemented, "method NodeListAndWatchResources not implemented")
-}
 
 func RegisterNodeServer(s *grpc.Server, srv NodeServer) {
 	s.RegisterService(&_Node_serviceDesc, srv)
@@ -728,27 +568,6 @@ func _Node_NodeUnprepareResources_Handler(srv interface{}, ctx context.Context,
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Node_NodeListAndWatchResources_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(NodeListAndWatchResourcesRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(NodeServer).NodeListAndWatchResources(m, &nodeNodeListAndWatchResourcesServer{stream})
-}
-
-type Node_NodeListAndWatchResourcesServer interface {
-	Send(*NodeListAndWatchResourcesResponse) error
-	grpc.ServerStream
-}
-
-type nodeNodeListAndWatchResourcesServer struct {
-	grpc.ServerStream
-}
-
-func (x *nodeNodeListAndWatchResourcesServer) Send(m *NodeListAndWatchResourcesResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
 var _Node_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "v1alpha3.Node",
 	HandlerType: (*NodeServer)(nil),
@@ -762,13 +581,7 @@ var _Node_serviceDesc = grpc.ServiceDesc{
 			Handler:    _Node_NodeUnprepareResources_Handler,
 		},
 	},
-	Streams: []grpc.StreamDesc{
+	Streams:  []grpc.StreamDesc{},
-		{
-			StreamName:    "NodeListAndWatchResources",
-			Handler:       _Node_NodeListAndWatchResources_Handler,
-			ServerStreams: true,
-		},
-	},
 	Metadata: "api.proto",
 }
 
@@ -1013,66 +826,6 @@ func (m *NodeUnprepareResourceResponse) MarshalToSizedBuffer(dAtA []byte) (int,
 	return len(dAtA) - i, nil
 }
 
-func (m *NodeListAndWatchResourcesRequest) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *NodeListAndWatchResourcesRequest) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *NodeListAndWatchResourcesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	return len(dAtA) - i, nil
-}
-
-func (m *NodeListAndWatchResourcesResponse) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *NodeListAndWatchResourcesResponse) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *NodeListAndWatchResourcesResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if len(m.Resources) > 0 {
-		for iNdEx := len(m.Resources) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Resources[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintApi(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0xa
-		}
-	}
-	return len(dAtA) - i, nil
-}
-
 func (m *Claim) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -1093,27 +846,6 @@ func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
-	if len(m.StructuredResourceHandle) > 0 {
-		for iNdEx := len(m.StructuredResourceHandle) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.StructuredResourceHandle[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintApi(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x2a
-		}
-	}
-	if len(m.ResourceHandle) > 0 {
-		i -= len(m.ResourceHandle)
-		copy(dAtA[i:], m.ResourceHandle)
-		i = encodeVarintApi(dAtA, i, uint64(len(m.ResourceHandle)))
-		i--
-		dAtA[i] = 0x22
-	}
 	if len(m.Name) > 0 {
 		i -= len(m.Name)
 		copy(dAtA[i:], m.Name)
@@ -1255,30 +987,6 @@ func (m *NodeUnprepareResourceResponse) Size() (n int) {
 	return n
 }
 
-func (m *NodeListAndWatchResourcesRequest) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	return n
-}
-
-func (m *NodeListAndWatchResourcesResponse) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	if len(m.Resources) > 0 {
-		for _, e := range m.Resources {
-			l = e.Size()
-			n += 1 + l + sovApi(uint64(l))
-		}
-	}
-	return n
-}
-
 func (m *Claim) Size() (n int) {
 	if m == nil {
 		return 0
@@ -1297,16 +1005,6 @@ func (m *Claim) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovApi(uint64(l))
 	}
-	l = len(m.ResourceHandle)
-	if l > 0 {
-		n += 1 + l + sovApi(uint64(l))
-	}
-	if len(m.StructuredResourceHandle) > 0 {
-		for _, e := range m.StructuredResourceHandle {
-			l = e.Size()
-			n += 1 + l + sovApi(uint64(l))
-		}
-	}
 	return n
 }
 
@@ -1407,45 +1105,14 @@ func (this *NodeUnprepareResourceResponse) String() string {
 	}, "")
 	return s
 }
-func (this *NodeListAndWatchResourcesRequest) String() string {
-	if this == nil {
-		return "nil"
-	}
-	s := strings.Join([]string{`&NodeListAndWatchResourcesRequest{`,
-		`}`,
-	}, "")
-	return s
-}
-func (this *NodeListAndWatchResourcesResponse) String() string {
-	if this == nil {
-		return "nil"
-	}
-	repeatedStringForResources := "[]*ResourceModel{"
-	for _, f := range this.Resources {
-		repeatedStringForResources += strings.Replace(fmt.Sprintf("%v", f), "ResourceModel", "v1alpha2.ResourceModel", 1) + ","
-	}
-	repeatedStringForResources += "}"
-	s := strings.Join([]string{`&NodeListAndWatchResourcesResponse{`,
-		`Resources:` + repeatedStringForResources + `,`,
-		`}`,
-	}, "")
-	return s
-}
 func (this *Claim) String() string {
 	if this == nil {
 		return "nil"
 	}
-	repeatedStringForStructuredResourceHandle := "[]*StructuredResourceHandle{"
-	for _, f := range this.StructuredResourceHandle {
-		repeatedStringForStructuredResourceHandle += strings.Replace(fmt.Sprintf("%v", f), "StructuredResourceHandle", "v1alpha2.StructuredResourceHandle", 1) + ","
-	}
-	repeatedStringForStructuredResourceHandle += "}"
 	s := strings.Join([]string{`&Claim{`,
 		`Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`,
 		`Uid:` + fmt.Sprintf("%v", this.Uid) + `,`,
 		`Name:` + fmt.Sprintf("%v", this.Name) + `,`,
-		`ResourceHandle:` + fmt.Sprintf("%v", this.ResourceHandle) + `,`,
-		`StructuredResourceHandle:` + repeatedStringForStructuredResourceHandle + `,`,
 		`}`,
 	}, "")
 	return s
@@ -2180,140 +1847,6 @@ func (m *NodeUnprepareResourceResponse) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
-func (m *NodeListAndWatchResourcesRequest) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowApi
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: NodeListAndWatchResourcesRequest: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: NodeListAndWatchResourcesRequest: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		default:
-			iNdEx = preIndex
-			skippy, err := skipApi(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthApi
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func (m *NodeListAndWatchResourcesResponse) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowApi
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: NodeListAndWatchResourcesResponse: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: NodeListAndWatchResourcesResponse: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Resources", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowApi
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthApi
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthApi
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Resources = append(m.Resources, &v1alpha2.ResourceModel{})
-			if err := m.Resources[len(m.Resources)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipApi(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthApi
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
 func (m *Claim) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
@@ -2439,72 +1972,6 @@ func (m *Claim) Unmarshal(dAtA []byte) error {
 			}
 			m.Name = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
-		case 4:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field ResourceHandle", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowApi
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthApi
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthApi
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.ResourceHandle = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 5:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field StructuredResourceHandle", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowApi
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthApi
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthApi
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.StructuredResourceHandle = append(m.StructuredResourceHandle, &v1alpha2.StructuredResourceHandle{})
-			if err := m.StructuredResourceHandle[len(m.StructuredResourceHandle)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipApi(dAtA[iNdEx:])

staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha4/api.proto

@@ -19,10 +19,9 @@ limitations under the License.
 syntax = "proto3";
 
 package v1alpha3;
-option go_package = "k8s.io/kubelet/pkg/apis/dra/v1alpha3";
+option go_package = "k8s.io/kubelet/pkg/apis/dra/v1alpha4";
 
 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
-import "k8s.io/api/resource/v1alpha2/generated.proto";
 
 option (gogoproto.goproto_stringer_all) = false;
 option (gogoproto.stringer_all) =  true;
@@ -44,12 +43,6 @@ service Node {
   // The same error handling rules apply,
   rpc NodeUnprepareResources (NodeUnprepareResourcesRequest)
     returns (NodeUnprepareResourcesResponse) {}
-
-  // NodeListAndWatchResources returns a stream of NodeResourcesResponse objects.
-  // At the start and whenever resource availability changes, the
-  // plugin must send one such object with all information to the Kubelet.
-  rpc NodeListAndWatchResources(NodeListAndWatchResourcesRequest)
-    returns (stream NodeListAndWatchResourcesResponse) {}
 }
 
 message NodePrepareResourcesRequest {
@@ -94,13 +87,6 @@ message NodeUnprepareResourceResponse {
     string error = 1;
 }
 
-message NodeListAndWatchResourcesRequest {
-}
-
-message NodeListAndWatchResourcesResponse {
-    repeated k8s.io.api.resource.v1alpha2.ResourceModel resources = 1;
-}
-
 message Claim {
     // The ResourceClaim namespace (ResourceClaim.meta.Namespace).
     // This field is REQUIRED.
@@ -111,13 +97,4 @@ message Claim {
     // The name of the Resource claim (ResourceClaim.meta.Name)
     // This field is REQUIRED.
     string name = 3;
-    // Resource handle (AllocationResult.ResourceHandles[*].Data)
-    // This field is REQUIRED.
-    string resource_handle = 4;
-    // Structured parameter resource handle (AllocationResult.ResourceHandles[*].StructuredData).
-    // This field is OPTIONAL. If present, it needs to be used
-    // instead of resource_handle. It will only have a single entry.
-    //
-    // Using "repeated" instead of "optional" is a workaround for https://github.com/gogo/protobuf/issues/713.
-    repeated k8s.io.api.resource.v1alpha2.StructuredResourceHandle structured_resource_handle = 5;
 }

test/e2e/dra/deploy.go

@@ -19,6 +19,7 @@ package dra
 import (
 	"bytes"
 	"context"
+	_ "embed"
 	"errors"
 	"fmt"
 	"net"
@@ -38,10 +39,19 @@ import (
 	v1 "k8s.io/api/core/v1"
 	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apiserver/pkg/authentication/serviceaccount"
+	"k8s.io/client-go/discovery/cached/memory"
 	resourceapiinformer "k8s.io/client-go/informers/resource/v1alpha2"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/dynamic-resource-allocation/kubeletplugin"
 	"k8s.io/klog/v2"
@@ -52,22 +62,26 @@ import (
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	"k8s.io/kubernetes/test/e2e/storage/drivers/proxy"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
+	"sigs.k8s.io/yaml"
 )
 
 const (
-	NodePrepareResourcesMethod      = "/v1alpha3.Node/NodePrepareResources"
+	NodePrepareResourcesMethod   = "/v1alpha3.Node/NodePrepareResources"
-	NodeUnprepareResourcesMethod    = "/v1alpha3.Node/NodeUnprepareResources"
+	NodeUnprepareResourcesMethod = "/v1alpha3.Node/NodeUnprepareResources"
-	NodeListAndWatchResourcesMethod = "/v1alpha3.Node/NodeListAndWatchResources"
 )
 
 type Nodes struct {
 	NodeNames []string
 }
 
+//go:embed test-driver/deploy/example/plugin-permissions.yaml
+var pluginPermissions string
+
 // NewNodes selects nodes to run the test on.
 func NewNodes(f *framework.Framework, minNodes, maxNodes int) *Nodes {
 	nodes := &Nodes{}
 	ginkgo.BeforeEach(func(ctx context.Context) {
+
 		ginkgo.By("selecting nodes")
 		// The kubelet plugin is harder. We deploy the builtin manifest
 		// after patching in the driver name and all nodes on which we
@@ -167,15 +181,19 @@ type MethodInstance struct {
 }
 
 type Driver struct {
-	f       *framework.Framework
+	f                  *framework.Framework
-	ctx     context.Context
+	ctx                context.Context
-	cleanup []func() // executed first-in-first-out
+	cleanup            []func() // executed first-in-first-out
-	wg      sync.WaitGroup
+	wg                 sync.WaitGroup
+	serviceAccountName string
 
 	NameSuffix string
 	Controller *app.ExampleController
 	Name       string
-	Nodes      map[string]*app.ExamplePlugin
+
+	// Nodes contains entries for each node selected for a test when the test runs.
+	// In addition, there is one entry for a fictional node.
+	Nodes map[string]KubeletPlugin
 
 	parameterMode         parameterMode
 	parameterAPIGroup     string
@@ -190,6 +208,11 @@ type Driver struct {
 	callCounts map[MethodInstance]int64
 }
 
+type KubeletPlugin struct {
+	*app.ExamplePlugin
+	ClientSet kubernetes.Interface
+}
+
 type parameterMode string
 
 const (
@@ -200,7 +223,7 @@ const (
 
 func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) {
 	ginkgo.By(fmt.Sprintf("deploying driver on nodes %v", nodes.NodeNames))
-	d.Nodes = map[string]*app.ExamplePlugin{}
+	d.Nodes = make(map[string]KubeletPlugin)
 	d.Name = d.f.UniqueName + d.NameSuffix + ".k8s.io"
 	resources.DriverName = d.Name
 
@@ -251,6 +274,13 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) {
 		framework.Failf("unknown test driver parameter mode: %s", d.parameterMode)
 	}
 
+	// Create service account and corresponding RBAC rules.
+	d.serviceAccountName = "dra-kubelet-plugin-" + d.Name + "-service-account"
+	content := pluginPermissions
+	content = strings.ReplaceAll(content, "dra-kubelet-plugin-namespace", d.f.Namespace.Name)
+	content = strings.ReplaceAll(content, "dra-kubelet-plugin", "dra-kubelet-plugin-"+d.Name)
+	d.createFromYAML(ctx, []byte(content), d.f.Namespace.Name)
+
 	instanceKey := "app.kubernetes.io/instance"
 	rsName := ""
 	draAddr := path.Join(framework.TestContext.KubeletRootDir, "plugins", d.Name+".sock")
@@ -263,6 +293,7 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) {
 			item.Spec.Replicas = &numNodes
 			item.Spec.Selector.MatchLabels[instanceKey] = d.Name
 			item.Spec.Template.Labels[instanceKey] = d.Name
+			item.Spec.Template.Spec.ServiceAccountName = d.serviceAccountName
 			item.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels[instanceKey] = d.Name
 			item.Spec.Template.Spec.Affinity.NodeAffinity = &v1.NodeAffinity{
 				RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
@@ -306,15 +337,31 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) {
 	framework.ExpectNoError(err, "list proxy pods")
 	gomega.Expect(numNodes).To(gomega.Equal(int32(len(pods.Items))), "number of proxy pods")
 
-	// Run registar and plugin for each of the pods.
+	// Run registrar and plugin for each of the pods.
 	for _, pod := range pods.Items {
 		// Need a local variable, not the loop variable, for the anonymous
 		// callback functions below.
 		pod := pod
 		nodename := pod.Spec.NodeName
+
+		// Authenticate the plugin so that it has the exact same
+		// permissions as the daemonset pod. This includes RBAC and a
+		// validating admission policy which limits writes to per-node
+		// ResourceSlices.
+		//
+		// We could retrieve
+		// /var/run/secrets/kubernetes.io/serviceaccount/token from
+		// each pod and use it. That would check that
+		// ServiceAccountTokenNodeBindingValidation works. But that's
+		// better covered by a test owned by SIG Auth (like the one in
+		// https://github.com/kubernetes/kubernetes/pull/124711).
+		//
+		// Here we merely use impersonation, which is faster.
+		driverClient := d.impersonateKubeletPlugin(&pod)
+
 		logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin"), "node", pod.Spec.NodeName, "pod", klog.KObj(&pod))
 		loggerCtx := klog.NewContext(ctx, logger)
-		plugin, err := app.StartPlugin(loggerCtx, "/cdi", d.Name, nodename,
+		plugin, err := app.StartPlugin(loggerCtx, "/cdi", d.Name, driverClient, nodename,
 			app.FileOperations{
 				Create: func(name string, content []byte) error {
 					klog.Background().Info("creating CDI file", "node", nodename, "filename", name, "content", string(content))
@@ -343,7 +390,7 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) {
 			// Depends on cancel being called first.
 			plugin.Stop()
 		})
-		d.Nodes[nodename] = plugin
+		d.Nodes[nodename] = KubeletPlugin{ExamplePlugin: plugin, ClientSet: driverClient}
 	}
 
 	// Wait for registration.
@@ -360,6 +407,26 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) {
 	}).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "hosts where the plugin has not been registered yet")
 }
 
+func (d *Driver) impersonateKubeletPlugin(pod *v1.Pod) kubernetes.Interface {
+	ginkgo.GinkgoHelper()
+	driverUserInfo := (&serviceaccount.ServiceAccountInfo{
+		Name:      d.serviceAccountName,
+		Namespace: pod.Namespace,
+		NodeName:  pod.Spec.NodeName,
+		PodName:   pod.Name,
+		PodUID:    string(pod.UID),
+	}).UserInfo()
+	driverClientConfig := d.f.ClientConfig()
+	driverClientConfig.Impersonate = rest.ImpersonationConfig{
+		UserName: driverUserInfo.GetName(),
+		Groups:   driverUserInfo.GetGroups(),
+		Extra:    driverUserInfo.GetExtra(),
+	}
+	driverClient, err := kubernetes.NewForConfig(driverClientConfig)
+	framework.ExpectNoError(err, "create client for driver")
+	return driverClient
+}
+
 func (d *Driver) createFile(pod *v1.Pod, name string, content []byte) error {
 	buffer := bytes.NewBuffer(content)
 	// Writing the content can be slow. Better create a temporary file and
@@ -376,6 +443,57 @@ func (d *Driver) removeFile(pod *v1.Pod, name string) error {
 	return d.podIO(pod).RemoveAll(name)
 }
 
+func (d *Driver) createFromYAML(ctx context.Context, content []byte, namespace string) {
+	// Not caching the discovery result isn't very efficient, but good enough.
+	discoveryCache := memory.NewMemCacheClient(d.f.ClientSet.Discovery())
+	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryCache)
+
+	for _, content := range bytes.Split(content, []byte("---\n")) {
+		if len(content) == 0 {
+			continue
+		}
+
+		var obj *unstructured.Unstructured
+		framework.ExpectNoError(yaml.UnmarshalStrict(content, &obj), fmt.Sprintf("Full YAML:\n%s\n", string(content)))
+
+		gv, err := schema.ParseGroupVersion(obj.GetAPIVersion())
+		framework.ExpectNoError(err, fmt.Sprintf("extract group+version from object %q", klog.KObj(obj)))
+		gk := schema.GroupKind{Group: gv.Group, Kind: obj.GetKind()}
+
+		mapping, err := restMapper.RESTMapping(gk, gv.Version)
+		framework.ExpectNoError(err, fmt.Sprintf("map %q to resource", gk))
+
+		resourceClient := d.f.DynamicClient.Resource(mapping.Resource)
+		options := metav1.CreateOptions{
+			// If the YAML input is invalid, then we want the
+			// apiserver to tell us via an error. This can
+			// happen because decoding into an unstructured object
+			// doesn't validate.
+			FieldValidation: "Strict",
+		}
+		switch mapping.Scope.Name() {
+		case meta.RESTScopeNameRoot:
+			_, err = resourceClient.Create(ctx, obj, options)
+		case meta.RESTScopeNameNamespace:
+			if namespace == "" {
+				framework.Failf("need namespace for object type %s", gk)
+			}
+			_, err = resourceClient.Namespace(namespace).Create(ctx, obj, options)
+		}
+		framework.ExpectNoError(err, "create object")
+		ginkgo.DeferCleanup(func(ctx context.Context) {
+			del := resourceClient.Delete
+			if mapping.Scope.Name() == meta.RESTScopeNameNamespace {
+				del = resourceClient.Namespace(namespace).Delete
+			}
+			err := del(ctx, obj.GetName(), metav1.DeleteOptions{})
+			if !apierrors.IsNotFound(err) {
+				framework.ExpectNoError(err, fmt.Sprintf("deleting %s.%s %s", obj.GetKind(), obj.GetAPIVersion(), klog.KObj(obj)))
+			}
+		})
+	}
+}
+
 func (d *Driver) podIO(pod *v1.Pod) proxy.PodDirIO {
 	logger := klog.Background()
 	return proxy.PodDirIO{

test/e2e/dra/dra.go

@@ -29,6 +29,7 @@ import (
 	"github.com/onsi/gomega"
 	"github.com/onsi/gomega/gcustom"
 	"github.com/onsi/gomega/gstruct"
+	"github.com/onsi/gomega/types"
 
 	v1 "k8s.io/api/core/v1"
 	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
@@ -891,17 +892,98 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 			driver := NewDriver(f, nodes, perNode(1, nodes))
 			driver.parameterMode = parameterModeStructured
 
-			f.It("must manage ResourceSlices", f.WithSlow(), func(ctx context.Context) {
+			f.It("must apply per-node permission checks", func(ctx context.Context) {
-				nodeName := nodes.NodeNames[0]
+				// All of the operations use the client set of a kubelet plugin for
-				driverName := driver.Name
+				// a fictional node which both don't exist, so nothing interferes
+				// when we actually manage to create a slice.
+				fictionalNodeName := "dra-fictional-node"
+				gomega.Expect(nodes.NodeNames).NotTo(gomega.ContainElement(fictionalNodeName))
+				fictionalNodeClient := driver.impersonateKubeletPlugin(&v1.Pod{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      fictionalNodeName + "-dra-plugin",
+						Namespace: f.Namespace.Name,
+						UID:       "12345",
+					},
+					Spec: v1.PodSpec{
+						NodeName: fictionalNodeName,
+					},
+				})
 
-				// Check for gRPC call on one node. If that already fails, then
+				// This is for some actual node in the cluster.
-				// we have a fundamental problem.
+				realNodeName := nodes.NodeNames[0]
-				m := MethodInstance{nodeName, NodeListAndWatchResourcesMethod}
+				realNodeClient := driver.Nodes[realNodeName].ClientSet
-				ginkgo.By("wait for NodeListAndWatchResources call")
+
-				gomega.Eventually(ctx, func() int64 {
+				// This is the slice that we try to create. It needs to be deleted
-					return driver.CallCount(m)
+				// after testing, if it still exists at that time.
-				}).WithTimeout(podStartTimeout).Should(gomega.BeNumerically(">", int64(0)), "NodeListAndWatchResources call count")
+				fictionalNodeSlice := &resourcev1alpha2.ResourceSlice{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: fictionalNodeName + "-slice",
+					},
+					NodeName:   fictionalNodeName,
+					DriverName: "dra.example.com",
+					ResourceModel: resourcev1alpha2.ResourceModel{
+						NamedResources: &resourcev1alpha2.NamedResourcesResources{},
+					},
+				}
+				ginkgo.DeferCleanup(func(ctx context.Context) {
+					err := f.ClientSet.ResourceV1alpha2().ResourceSlices().Delete(ctx, fictionalNodeSlice.Name, metav1.DeleteOptions{})
+					if !apierrors.IsNotFound(err) {
+						framework.ExpectNoError(err)
+					}
+				})
+
+				// Message from test-driver/deploy/example/plugin-permissions.yaml
+				matchVAPDeniedError := gomega.MatchError(gomega.ContainSubstring("may only modify resourceslices that belong to the node the pod is running on"))
+
+				mustCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourcev1alpha2.ResourceSlice) *resourcev1alpha2.ResourceSlice {
+					ginkgo.GinkgoHelper()
+					slice, err := clientSet.ResourceV1alpha2().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{})
+					framework.ExpectNoError(err, fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name))
+					return slice
+				}
+				mustUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourcev1alpha2.ResourceSlice) *resourcev1alpha2.ResourceSlice {
+					ginkgo.GinkgoHelper()
+					slice, err := clientSet.ResourceV1alpha2().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{})
+					framework.ExpectNoError(err, fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name))
+					return slice
+				}
+				mustDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourcev1alpha2.ResourceSlice) {
+					ginkgo.GinkgoHelper()
+					err := clientSet.ResourceV1alpha2().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{})
+					framework.ExpectNoError(err, fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name))
+				}
+				mustFailToCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourcev1alpha2.ResourceSlice, matchError types.GomegaMatcher) {
+					ginkgo.GinkgoHelper()
+					_, err := clientSet.ResourceV1alpha2().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{})
+					gomega.Expect(err).To(matchError, fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name))
+				}
+				mustFailToUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourcev1alpha2.ResourceSlice, matchError types.GomegaMatcher) {
+					ginkgo.GinkgoHelper()
+					_, err := clientSet.ResourceV1alpha2().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{})
+					gomega.Expect(err).To(matchError, fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name))
+				}
+				mustFailToDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourcev1alpha2.ResourceSlice, matchError types.GomegaMatcher) {
+					ginkgo.GinkgoHelper()
+					err := clientSet.ResourceV1alpha2().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{})
+					gomega.Expect(err).To(matchError, fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name))
+				}
+
+				// Create with different clients, keep it in the end.
+				mustFailToCreate(realNodeClient, "real plugin", fictionalNodeSlice, matchVAPDeniedError)
+				createdFictionalNodeSlice := mustCreate(fictionalNodeClient, "fictional plugin", fictionalNodeSlice)
+
+				// Update with different clients.
+				mustFailToUpdate(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError)
+				createdFictionalNodeSlice = mustUpdate(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice)
+				createdFictionalNodeSlice = mustUpdate(f.ClientSet, "admin", createdFictionalNodeSlice)
+
+				// Delete with different clients.
+				mustFailToDelete(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError)
+				mustDelete(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice)
+			})
+
+			f.It("must manage ResourceSlices", f.WithSlow(), func(ctx context.Context) {
+				driverName := driver.Name
 
 				// Now check for exactly the right set of objects for all nodes.
 				ginkgo.By("check if ResourceSlice object(s) exist on the API server")
@@ -1109,6 +1191,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 			b := newBuilder(f, driver)
 			preScheduledTests(b, driver, resourcev1alpha2.AllocationModeImmediate)
 			claimTests(b, driver, resourcev1alpha2.AllocationModeImmediate)
+
 		})
 	})
 

test/e2e/dra/test-driver/README.md

@@ -66,7 +66,7 @@ go run ./test/e2e/dra/test-driver --feature-gates ContextualLogging=true -v=5 co
 In yet another:
 ```console
 sudo mkdir -p /var/run/cdi && sudo chmod a+rwx /var/run/cdi /var/lib/kubelet/plugins_registry
-go run ./test/e2e/dra/test-driver --feature-gates ContextualLogging=true -v=5 kubelet-plugin
+go run ./test/e2e/dra/test-driver --feature-gates ContextualLogging=true -v=5 kubelet-plugin --node-name=127.0.0.1
 ```
 
 And finally:

test/e2e/dra/test-driver/app/kubeletplugin.go

@@ -26,24 +26,25 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 
 	resourceapi "k8s.io/api/resource/v1alpha2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/dynamic-resource-allocation/kubeletplugin"
 	"k8s.io/klog/v2"
-	drapbv1alpha3 "k8s.io/kubelet/pkg/apis/dra/v1alpha3"
+	drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4"
 )
 
 type ExamplePlugin struct {
-	stopCh  <-chan struct{}
+	stopCh     <-chan struct{}
-	logger  klog.Logger
+	logger     klog.Logger
-	d       kubeletplugin.DRAPlugin
+	kubeClient kubernetes.Interface
-	fileOps FileOperations
+	d          kubeletplugin.DRAPlugin
+	fileOps    FileOperations
 
 	cdiDir     string
 	driverName string
@@ -52,7 +53,7 @@ type ExamplePlugin struct {
 
 	mutex          sync.Mutex
 	instancesInUse sets.Set[string]
-	prepared       map[ClaimID]any
+	prepared       map[ClaimID][]string // instance names
 	gRPCCalls      []GRPCCall
 
 	blockPrepareResourcesMutex   sync.Mutex
@@ -87,7 +88,7 @@ type ClaimID struct {
 	UID  string
 }
 
-var _ drapbv1alpha3.NodeServer = &ExamplePlugin{}
+var _ drapb.NodeServer = &ExamplePlugin{}
 
 // getJSONFilePath returns the absolute path where CDI file is/should be.
 func (ex *ExamplePlugin) getJSONFilePath(claimUID string) string {
@@ -111,8 +112,9 @@ type FileOperations struct {
 }
 
 // StartPlugin sets up the servers that are necessary for a DRA kubelet plugin.
-func StartPlugin(ctx context.Context, cdiDir, driverName string, nodeName string, fileOps FileOperations, opts ...kubeletplugin.Option) (*ExamplePlugin, error) {
+func StartPlugin(ctx context.Context, cdiDir, driverName string, kubeClient kubernetes.Interface, nodeName string, fileOps FileOperations, opts ...kubeletplugin.Option) (*ExamplePlugin, error) {
 	logger := klog.FromContext(ctx)
+
 	if fileOps.Create == nil {
 		fileOps.Create = func(name string, content []byte) error {
 			return os.WriteFile(name, content, os.FileMode(0644))
@@ -129,13 +131,14 @@ func StartPlugin(ctx context.Context, cdiDir, driverName string, nodeName string
 	ex := &ExamplePlugin{
 		stopCh:         ctx.Done(),
 		logger:         logger,
+		kubeClient:     kubeClient,
 		fileOps:        fileOps,
 		cdiDir:         cdiDir,
 		driverName:     driverName,
 		nodeName:       nodeName,
 		instances:      sets.New[string](),
 		instancesInUse: sets.New[string](),
-		prepared:       make(map[ClaimID]any),
+		prepared:       make(map[ClaimID][]string),
 	}
 
 	for i := 0; i < ex.fileOps.NumResourceInstances; i++ {
@@ -143,17 +146,33 @@ func StartPlugin(ctx context.Context, cdiDir, driverName string, nodeName string
 	}
 
 	opts = append(opts,
-		kubeletplugin.Logger(logger),
 		kubeletplugin.DriverName(driverName),
+		kubeletplugin.NodeName(nodeName),
+		kubeletplugin.KubeClient(kubeClient),
 		kubeletplugin.GRPCInterceptor(ex.recordGRPCCall),
 		kubeletplugin.GRPCStreamInterceptor(ex.recordGRPCStream),
 	)
-	d, err := kubeletplugin.Start(ex, opts...)
+	d, err := kubeletplugin.Start(ctx, ex, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("start kubelet plugin: %w", err)
 	}
 	ex.d = d
 
+	if fileOps.NumResourceInstances >= 0 {
+		instances := make([]resourceapi.NamedResourcesInstance, ex.fileOps.NumResourceInstances)
+		for i := 0; i < ex.fileOps.NumResourceInstances; i++ {
+			instances[i].Name = fmt.Sprintf("instance-%02d", i)
+		}
+		nodeResources := []*resourceapi.ResourceModel{
+			{
+				NamedResources: &resourceapi.NamedResourcesResources{
+					Instances: instances,
+				},
+			},
+		}
+		ex.d.PublishResources(ctx, nodeResources)
+	}
+
 	return ex, nil
 }
 
@@ -230,19 +249,47 @@ func (ex *ExamplePlugin) getUnprepareResourcesFailure() error {
 // a deterministic name to simplify NodeUnprepareResource (no need to remember
 // or discover the name) and idempotency (when called again, the file simply
 // gets written again).
-func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName string, claimUID string, resourceHandle string, structuredResourceHandle []*resourceapi.StructuredResourceHandle) ([]string, error) {
+func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drapb.Claim) ([]string, error) {
 	logger := klog.FromContext(ctx)
 
+	// The plugin must retrieve the claim itself to get it in the version
+	// that it understands.
+	var resourceHandle string
+	var structuredResourceHandle *resourceapi.StructuredResourceHandle
+	claim, err := ex.kubeClient.ResourceV1alpha2().ResourceClaims(claimReq.Namespace).Get(ctx, claimReq.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("retrieve claim %s/%s: %w", claimReq.Namespace, claimReq.Name, err)
+	}
+	if claim.Status.Allocation == nil {
+		return nil, fmt.Errorf("claim %s/%s not allocated", claimReq.Namespace, claimReq.Name)
+	}
+	if claim.UID != types.UID(claimReq.Uid) {
+		return nil, fmt.Errorf("claim %s/%s got replaced", claimReq.Namespace, claimReq.Name)
+	}
+	haveResources := false
+	for _, handle := range claim.Status.Allocation.ResourceHandles {
+		if handle.DriverName == ex.driverName {
+			haveResources = true
+			resourceHandle = handle.Data
+			structuredResourceHandle = handle.StructuredData
+			break
+		}
+	}
+	if !haveResources {
+		// Nothing to do.
+		return nil, nil
+	}
+
 	ex.mutex.Lock()
 	defer ex.mutex.Unlock()
 	ex.blockPrepareResourcesMutex.Lock()
 	defer ex.blockPrepareResourcesMutex.Unlock()
 
-	deviceName := "claim-" + claimUID
+	deviceName := "claim-" + claimReq.Uid
 	vendor := ex.driverName
 	class := "test"
 	dev := vendor + "/" + class + "=" + deviceName
-	claimID := ClaimID{Name: claimName, UID: claimUID}
+	claimID := ClaimID{Name: claimReq.Name, UID: claimReq.Uid}
 	if _, ok := ex.prepared[claimID]; ok {
 		// Idempotent call, nothing to do.
 		return []string{dev}, nil
@@ -250,29 +297,22 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri
 
 	// Determine environment variables.
 	var p parameters
-	var actualResourceHandle any
 	var instanceNames []string
-	switch len(structuredResourceHandle) {
+	if structuredResourceHandle == nil {
-	case 0:
 		// Control plane controller did the allocation.
 		if err := json.Unmarshal([]byte(resourceHandle), &p); err != nil {
 			return nil, fmt.Errorf("unmarshal resource handle: %w", err)
 		}
-		actualResourceHandle = resourceHandle
+	} else {
-	case 1:
 		// Scheduler did the allocation with structured parameters.
-		handle := structuredResourceHandle[0]
+		p.NodeName = structuredResourceHandle.NodeName
-		if handle == nil {
+		if err := extractParameters(structuredResourceHandle.VendorClassParameters, &p.EnvVars, "admin"); err != nil {
-			return nil, errors.New("unexpected nil StructuredResourceHandle")
-		}
-		p.NodeName = handle.NodeName
-		if err := extractParameters(handle.VendorClassParameters, &p.EnvVars, "admin"); err != nil {
 			return nil, err
 		}
-		if err := extractParameters(handle.VendorClaimParameters, &p.EnvVars, "user"); err != nil {
+		if err := extractParameters(structuredResourceHandle.VendorClaimParameters, &p.EnvVars, "user"); err != nil {
 			return nil, err
 		}
-		for _, result := range handle.Results {
+		for _, result := range structuredResourceHandle.Results {
 			if err := extractParameters(result.VendorRequestParameters, &p.EnvVars, "user"); err != nil {
 				return nil, err
 			}
@@ -292,10 +332,6 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri
 			}
 			instanceNames = append(instanceNames, instanceName)
 		}
-		actualResourceHandle = handle
-	default:
-		// Huh?
-		return nil, fmt.Errorf("invalid length of NodePrepareResourceRequest.StructuredResourceHandle: %d", len(structuredResourceHandle))
 	}
 
 	// Sanity check scheduling.
@@ -323,7 +359,7 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri
 			},
 		},
 	}
-	filePath := ex.getJSONFilePath(claimUID)
+	filePath := ex.getJSONFilePath(claimReq.Uid)
 	buffer, err := json.Marshal(spec)
 	if err != nil {
 		return nil, fmt.Errorf("marshal spec: %w", err)
@@ -332,7 +368,7 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri
 		return nil, fmt.Errorf("failed to write CDI file %v", err)
 	}
 
-	ex.prepared[claimID] = actualResourceHandle
+	ex.prepared[claimID] = instanceNames
 	for _, instanceName := range instanceNames {
 		ex.instancesInUse.Insert(instanceName)
 	}
@@ -358,9 +394,9 @@ func extractParameters(parameters runtime.RawExtension, env *map[string]string,
 	return nil
 }
 
-func (ex *ExamplePlugin) NodePrepareResources(ctx context.Context, req *drapbv1alpha3.NodePrepareResourcesRequest) (*drapbv1alpha3.NodePrepareResourcesResponse, error) {
+func (ex *ExamplePlugin) NodePrepareResources(ctx context.Context, req *drapb.NodePrepareResourcesRequest) (*drapb.NodePrepareResourcesResponse, error) {
-	resp := &drapbv1alpha3.NodePrepareResourcesResponse{
+	resp := &drapb.NodePrepareResourcesResponse{
-		Claims: make(map[string]*drapbv1alpha3.NodePrepareResourceResponse),
+		Claims: make(map[string]*drapb.NodePrepareResourceResponse),
 	}
 
 	if failure := ex.getPrepareResourcesFailure(); failure != nil {
@@ -368,13 +404,13 @@ func (ex *ExamplePlugin) NodePrepareResources(ctx context.Context, req *drapbv1a
 	}
 
 	for _, claimReq := range req.Claims {
-		cdiDevices, err := ex.nodePrepareResource(ctx, claimReq.Name, claimReq.Uid, claimReq.ResourceHandle, claimReq.StructuredResourceHandle)
+		cdiDevices, err := ex.nodePrepareResource(ctx, claimReq)
 		if err != nil {
-			resp.Claims[claimReq.Uid] = &drapbv1alpha3.NodePrepareResourceResponse{
+			resp.Claims[claimReq.Uid] = &drapb.NodePrepareResourceResponse{
 				Error: err.Error(),
 			}
 		} else {
-			resp.Claims[claimReq.Uid] = &drapbv1alpha3.NodePrepareResourceResponse{
+			resp.Claims[claimReq.Uid] = &drapb.NodePrepareResourceResponse{
 				CDIDevices: cdiDevices,
 			}
 		}
@@ -385,13 +421,13 @@ func (ex *ExamplePlugin) NodePrepareResources(ctx context.Context, req *drapbv1a
 // NodeUnprepareResource removes the CDI file created by
 // NodePrepareResource. It's idempotent, therefore it is not an error when that
 // file is already gone.
-func (ex *ExamplePlugin) nodeUnprepareResource(ctx context.Context, claimName string, claimUID string, resourceHandle string, structuredResourceHandle []*resourceapi.StructuredResourceHandle) error {
+func (ex *ExamplePlugin) nodeUnprepareResource(ctx context.Context, claimReq *drapb.Claim) error {
 	ex.blockUnprepareResourcesMutex.Lock()
 	defer ex.blockUnprepareResourcesMutex.Unlock()
 
 	logger := klog.FromContext(ctx)
 
-	filePath := ex.getJSONFilePath(claimUID)
+	filePath := ex.getJSONFilePath(claimReq.Uid)
 	if err := ex.fileOps.Remove(filePath); err != nil {
 		return fmt.Errorf("error removing CDI file: %w", err)
 	}
@@ -400,40 +436,24 @@ func (ex *ExamplePlugin) nodeUnprepareResource(ctx context.Context, claimName st
 	ex.mutex.Lock()
 	defer ex.mutex.Unlock()
 
-	claimID := ClaimID{Name: claimName, UID: claimUID}
+	claimID := ClaimID{Name: claimReq.Name, UID: claimReq.Uid}
-	expectedResourceHandle, ok := ex.prepared[claimID]
+	instanceNames, ok := ex.prepared[claimID]
 	if !ok {
 		// Idempotent call, nothing to do.
 		return nil
 	}
 
-	var actualResourceHandle any = resourceHandle
-	if structuredResourceHandle != nil {
-		if len(structuredResourceHandle) != 1 {
-			return fmt.Errorf("unexpected number of entries in StructuredResourceHandle: %d", len(structuredResourceHandle))
-		}
-		actualResourceHandle = structuredResourceHandle[0]
-	}
-	if diff := cmp.Diff(expectedResourceHandle, actualResourceHandle); diff != "" {
-		return fmt.Errorf("difference between expected (-) and actual resource handle (+):\n%s", diff)
-	}
 	delete(ex.prepared, claimID)
-	if structuredResourceHandle := structuredResourceHandle; structuredResourceHandle != nil {
+	for _, instanceName := range instanceNames {
-		for _, handle := range structuredResourceHandle {
+		ex.instancesInUse.Delete(instanceName)
-			for _, result := range handle.Results {
-				instanceName := result.NamedResources.Name
-				ex.instancesInUse.Delete(instanceName)
-			}
-		}
 	}
-	delete(ex.prepared, ClaimID{Name: claimName, UID: claimUID})
 
 	return nil
 }
 
-func (ex *ExamplePlugin) NodeUnprepareResources(ctx context.Context, req *drapbv1alpha3.NodeUnprepareResourcesRequest) (*drapbv1alpha3.NodeUnprepareResourcesResponse, error) {
+func (ex *ExamplePlugin) NodeUnprepareResources(ctx context.Context, req *drapb.NodeUnprepareResourcesRequest) (*drapb.NodeUnprepareResourcesResponse, error) {
-	resp := &drapbv1alpha3.NodeUnprepareResourcesResponse{
+	resp := &drapb.NodeUnprepareResourcesResponse{
-		Claims: make(map[string]*drapbv1alpha3.NodeUnprepareResourceResponse),
+		Claims: make(map[string]*drapb.NodeUnprepareResourceResponse),
 	}
 
 	if failure := ex.getUnprepareResourcesFailure(); failure != nil {
@@ -441,51 +461,18 @@ func (ex *ExamplePlugin) NodeUnprepareResources(ctx context.Context, req *drapbv
 	}
 
 	for _, claimReq := range req.Claims {
-		err := ex.nodeUnprepareResource(ctx, claimReq.Name, claimReq.Uid, claimReq.ResourceHandle, claimReq.StructuredResourceHandle)
+		err := ex.nodeUnprepareResource(ctx, claimReq)
 		if err != nil {
-			resp.Claims[claimReq.Uid] = &drapbv1alpha3.NodeUnprepareResourceResponse{
+			resp.Claims[claimReq.Uid] = &drapb.NodeUnprepareResourceResponse{
 				Error: err.Error(),
 			}
 		} else {
-			resp.Claims[claimReq.Uid] = &drapbv1alpha3.NodeUnprepareResourceResponse{}
+			resp.Claims[claimReq.Uid] = &drapb.NodeUnprepareResourceResponse{}
 		}
 	}
 	return resp, nil
 }
 
-func (ex *ExamplePlugin) NodeListAndWatchResources(req *drapbv1alpha3.NodeListAndWatchResourcesRequest, stream drapbv1alpha3.Node_NodeListAndWatchResourcesServer) error {
-	if ex.fileOps.NumResourceInstances < 0 {
-		ex.logger.Info("Sending no NodeResourcesResponse")
-		return status.New(codes.Unimplemented, "node resource support disabled").Err()
-	}
-
-	instances := make([]resourceapi.NamedResourcesInstance, len(ex.instances))
-	for i, name := range sets.List(ex.instances) {
-		instances[i].Name = name
-	}
-	resp := &drapbv1alpha3.NodeListAndWatchResourcesResponse{
-		Resources: []*resourceapi.ResourceModel{
-			{
-				NamedResources: &resourceapi.NamedResourcesResources{
-					Instances: instances,
-				},
-			},
-		},
-	}
-
-	ex.logger.Info("Sending NodeListAndWatchResourcesResponse", "response", resp)
-	if err := stream.Send(resp); err != nil {
-		return err
-	}
-
-	// Keep the stream open until the test is done.
-	// TODO: test sending more updates later
-	<-ex.stopCh
-	ex.logger.Info("Done sending NodeListAndWatchResourcesResponse, closing stream")
-
-	return nil
-}
-
 func (ex *ExamplePlugin) GetPreparedResources() []ClaimID {
 	ex.mutex.Lock()
 	defer ex.mutex.Unlock()

test/e2e/dra/test-driver/app/server.go

@@ -21,6 +21,7 @@ package app
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -272,6 +273,7 @@ func NewCommand() *cobra.Command {
 	draAddress := fs.String("dra-address", "/var/lib/kubelet/plugins/test-driver/dra.sock", "The Unix domain socket that kubelet will connect to for dynamic resource allocation requests, in the filesystem of kubelet.")
 	fs = kubeletPluginFlagSets.FlagSet("CDI")
 	cdiDir := fs.String("cdi-dir", "/var/run/cdi", "directory for dynamically created CDI JSON files")
+	nodeName := fs.String("node-name", "", "name of the node that the kubelet plugin is responsible for")
 	fs = kubeletPlugin.Flags()
 	for _, f := range kubeletPluginFlagSets.FlagSets {
 		fs.AddFlagSet(f)
@@ -287,7 +289,11 @@ func NewCommand() *cobra.Command {
 			return fmt.Errorf("create socket directory: %w", err)
 		}
 
-		plugin, err := StartPlugin(cmd.Context(), *cdiDir, *driverName, "", FileOperations{},
+		if *nodeName == "" {
+			return errors.New("--node-name not set")
+		}
+
+		plugin, err := StartPlugin(cmd.Context(), *cdiDir, *driverName, clientset, *nodeName, FileOperations{},
 			kubeletplugin.PluginSocketPath(*endpoint),
 			kubeletplugin.RegistrarSocketPath(path.Join(*pluginRegistrationPath, *driverName+"-reg.sock")),
 			kubeletplugin.KubeletPluginSocketPath(*draAddress),

test/e2e/dra/test-driver/deploy/example/plugin-permissions.yaml

@@ -0,0 +1,84 @@
+# Real driver deployments must replace all occurrences of "dra-kubelet-plugin"
+# with something specific to their driver.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dra-kubelet-plugin-service-account
+  namespace: dra-kubelet-plugin-namespace
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: dra-kubelet-plugin-role
+rules:
+- apiGroups: ["resource.k8s.io"]
+  resources: ["resourceclaims"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get"]
+- apiGroups: ["resource.k8s.io"]
+  resources: ["resourceslices"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: dra-kubelet-plugin-role-binding
+subjects:
+- kind: ServiceAccount
+  name: dra-kubelet-plugin-service-account
+  namespace: dra-kubelet-plugin-namespace
+roleRef:
+  kind: ClusterRole
+  name: dra-kubelet-plugin-role
+  apiGroup: rbac.authorization.k8s.io
+---
+# This ValidatingAdmissionPolicy is specific to the DRA driver's kubelet plugin
+# because it checks the ServiceAccount defined for it above. An admin could
+# also define a single policy for all drivers.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: resourceslices-policy-dra-kubelet-plugin
+spec:
+  failurePolicy: Fail
+  matchConstraints:
+    resourceRules:
+    - apiGroups:   ["resource.k8s.io"]
+      apiVersions: ["v1alpha2"]
+      operations:  ["CREATE", "UPDATE", "DELETE"]
+      resources:   ["resourceslices"]
+  variables:
+  - name: hasNodeName
+    expression: >-
+      "authentication.kubernetes.io/node-name" in request.userInfo.extra
+  - name: isKubeletPlugin
+    expression: >-
+      request.userInfo.username == "system:serviceaccount:dra-kubelet-plugin-namespace:dra-kubelet-plugin-service-account"
+  - name: objectNodeName
+    expression: >-
+      (request.operation == "DELETE" ? oldObject : object).?nodeName.orValue("")
+  validations:
+  - expression: >-
+      !variables.isKubeletPlugin || variables.hasNodeName
+    message: This user must have a "authentication.kubernetes.io/node-name" claim. ServiceAccountTokenNodeBindingValidation must be enabled in the cluster.
+  - expression: >-
+      !variables.isKubeletPlugin || !variables.hasNodeName ||
+      variables.objectNodeName == request.userInfo.extra["authentication.kubernetes.io/node-name"][0]
+    message: This DRA kubelet plugin may only modify resourceslices that belong to the node the pod is running on.
+    # This is useful for debugging. Can be dropped in a production deployment.
+    messageExpression: >-
+      "The DRA kubelet plugin on node " + request.userInfo.extra["authentication.kubernetes.io/node-name"][0] +
+      " may only modify resourceslices that belong to the node the pod is running on, not " +
+      (variables.objectNodeName == "" ? variables.objectNodeName : "a cluster-scoped slice") + "."
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: resourceslices-policy-dra-kubelet-plugin
+spec:
+  policyName: resourceslices-policy-dra-kubelet-plugin
+  validationActions: [Deny]
+  # All ResourceSlices are matched.

test/e2e_node/dra_test.go

@@ -34,9 +34,12 @@ import (
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gstruct"
+	"github.com/onsi/gomega/types"
 
 	v1 "k8s.io/api/core/v1"
 	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
@@ -66,14 +69,19 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 	f := framework.NewDefaultFramework("dra-node")
 	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
 
-	var kubeletPlugin, kubeletPlugin1, kubeletPlugin2 *testdriver.ExamplePlugin
+	ginkgo.BeforeEach(func() {
+		ginkgo.DeferCleanup(func(ctx context.Context) {
+			// When plugin and kubelet get killed at the end of the tests, they leave ResourceSlices behind.
+			// Perhaps garbage collection would eventually remove them (not sure how the node instance
+			// is managed), but this could take time. Let's clean up explicitly.
+			framework.ExpectNoError(f.ClientSet.ResourceV1alpha2().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}))
+		})
+	})
 
 	f.Context("Resource Kubelet Plugin", f.WithSerial(), func() {
-		ginkgo.BeforeEach(func(ctx context.Context) {
-			kubeletPlugin = newKubeletPlugin(ctx, getNodeName(ctx, f), driverName)
-		})
-
 		ginkgo.It("must register after Kubelet restart", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			oldCalls := kubeletPlugin.GetGRPCCalls()
 			getNewCalls := func() []testdriver.GRPCCall {
 				calls := kubeletPlugin.GetGRPCCalls()
@@ -88,16 +96,21 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must register after plugin restart", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			ginkgo.By("restart Kubelet Plugin")
 			kubeletPlugin.Stop()
-			kubeletPlugin = newKubeletPlugin(ctx, getNodeName(ctx, f), driverName)
+			kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
 
 			ginkgo.By("wait for Kubelet plugin re-registration")
 			gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
 		})
 
 		ginkgo.It("must process pod created when kubelet is not running", func(ctx context.Context) {
+			newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			// Stop Kubelet
+			ginkgo.By("stop kubelet")
 			startKubelet := stopKubelet()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
 			// Pod must be in pending state
@@ -106,6 +119,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 			})
 			framework.ExpectNoError(err)
 			// Start Kubelet
+			ginkgo.By("restart kubelet")
 			startKubelet()
 			// Pod should succeed
 			err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.Name, f.Namespace.Name, framework.PodStartShortTimeout)
@@ -113,6 +127,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must keep pod in pending state if NodePrepareResources times out", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unblock := kubeletPlugin.BlockNodePrepareResources()
 			defer unblock()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
@@ -131,6 +147,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must run pod if NodePrepareResources fails and then succeeds", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unset := kubeletPlugin.SetNodePrepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
 
@@ -154,6 +172,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must run pod if NodeUnprepareResources fails and then succeeds", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unset := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
 
@@ -174,6 +194,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must retry NodePrepareResources after Kubelet restart", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unset := kubeletPlugin.SetNodePrepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
 
@@ -203,6 +225,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must retry NodeUnprepareResources after Kubelet restart", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unset := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
 			ginkgo.By("wait for NodePrepareResources call to succeed")
@@ -228,6 +252,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must call NodeUnprepareResources for deleted pod", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unset := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName})
 
@@ -250,6 +276,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must call NodeUnprepareResources for deleted pod after Kubelet restart", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unset := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName})
 
@@ -279,6 +307,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must not call NodePrepareResources for deleted pod after Kubelet restart", func(ctx context.Context) {
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unblock := kubeletPlugin.BlockNodePrepareResources()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName})
 
@@ -306,16 +336,21 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 	})
 
 	f.Context("Two resource Kubelet Plugins", f.WithSerial(), func() {
-		ginkgo.BeforeEach(func(ctx context.Context) {
+		// start creates plugins which will get stopped when the context gets canceled.
-			kubeletPlugin1 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin1Name)
+		start := func(ctx context.Context) (*testdriver.ExamplePlugin, *testdriver.ExamplePlugin) {
-			kubeletPlugin2 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin2Name)
+			kubeletPlugin1 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin1Name)
+			kubeletPlugin2 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin2Name)
 
 			ginkgo.By("wait for Kubelet plugin registration")
 			gomega.Eventually(kubeletPlugin1.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
 			gomega.Eventually(kubeletPlugin2.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
-		})
+
+			return kubeletPlugin1, kubeletPlugin2
+		}
 
 		ginkgo.It("must prepare and unprepare resources", func(ctx context.Context) {
+			kubeletPlugin1, kubeletPlugin2 := start(ctx)
+
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
 
 			ginkgo.By("wait for pod to succeed")
@@ -332,6 +367,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must run pod if NodePrepareResources fails for one plugin and then succeeds", func(ctx context.Context) {
+			_, kubeletPlugin2 := start(ctx)
+
 			unset := kubeletPlugin2.SetNodePrepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
 
@@ -355,6 +392,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must run pod if NodeUnprepareResources fails for one plugin and then succeeds", func(ctx context.Context) {
+			kubeletPlugin1, kubeletPlugin2 := start(ctx)
+
 			unset := kubeletPlugin2.SetNodeUnprepareResourcesFailureMode()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
 
@@ -378,6 +417,9 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must run pod if NodePrepareResources is in progress for one plugin when Kubelet restarts", func(ctx context.Context) {
+			_, kubeletPlugin2 := start(ctx)
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+
 			unblock := kubeletPlugin.BlockNodePrepareResources()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
 
@@ -401,6 +443,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 		})
 
 		ginkgo.It("must call NodeUnprepareResources again if it's in progress for one plugin when Kubelet restarts", func(ctx context.Context) {
+			kubeletPlugin1, kubeletPlugin2 := start(ctx)
+
 			unblock := kubeletPlugin2.BlockNodeUnprepareResources()
 			pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{kubeletPlugin1Name, kubeletPlugin2Name})
 
@@ -423,10 +467,70 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 			framework.ExpectNoError(err)
 		})
 	})
+
+	f.Context("ResourceSlice", f.WithSerial(), func() {
+		listResources := func(ctx context.Context) ([]resourcev1alpha2.ResourceSlice, error) {
+			slices, err := f.ClientSet.ResourceV1alpha2().ResourceSlices().List(ctx, metav1.ListOptions{})
+			if err != nil {
+				return nil, err
+			}
+			return slices.Items, nil
+		}
+
+		matchResourcesByNodeName := func(nodeName string) types.GomegaMatcher {
+			return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+				"NodeName": gomega.Equal(nodeName),
+			})
+		}
+
+		f.It("must be removed on kubelet startup", f.WithDisruptive(), func(ctx context.Context) {
+			ginkgo.By("stop kubelet")
+			startKubelet := stopKubelet()
+			ginkgo.DeferCleanup(func() {
+				if startKubelet != nil {
+					startKubelet()
+				}
+			})
+
+			ginkgo.By("create some ResourceSlices")
+			nodeName := getNodeName(ctx, f)
+			otherNodeName := nodeName + "-other"
+			createTestResourceSlice(ctx, f.ClientSet, nodeName, driverName)
+			createTestResourceSlice(ctx, f.ClientSet, nodeName+"-other", driverName)
+
+			matchAll := gomega.ConsistOf(matchResourcesByNodeName(nodeName), matchResourcesByNodeName(otherNodeName))
+			matchOtherNode := gomega.ConsistOf(matchResourcesByNodeName(otherNodeName))
+
+			gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchAll, "ResourceSlices without kubelet")
+
+			ginkgo.By("start kubelet")
+			startKubelet()
+			startKubelet = nil
+
+			ginkgo.By("wait for exactly the node's ResourceSlice to get deleted")
+			gomega.Eventually(ctx, listResources).Should(matchOtherNode, "ResourceSlices with kubelet")
+			gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchOtherNode, "ResourceSlices with kubelet")
+		})
+
+		f.It("must be removed after plugin unregistration", func(ctx context.Context) {
+			nodeName := getNodeName(ctx, f)
+			matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName))
+
+			ginkgo.By("start plugin and wait for ResourceSlice")
+			kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
+			gomega.Eventually(ctx, listResources).Should(matchNode, "ResourceSlice from kubelet plugin")
+			gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchNode, "ResourceSlice from kubelet plugin")
+
+			ginkgo.By("stop plugin and wait for ResourceSlice removal")
+			kubeletPlugin.Stop()
+			gomega.Eventually(ctx, listResources).Should(gomega.BeEmpty(), "ResourceSlices with no plugin")
+			gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(gomega.BeEmpty(), "ResourceSlices with no plugin")
+		})
+	})
 })
 
 // Run Kubelet plugin and wait until it's registered
-func newKubeletPlugin(ctx context.Context, nodeName, pluginName string) *testdriver.ExamplePlugin {
+func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeName, pluginName string) *testdriver.ExamplePlugin {
 	ginkgo.By("start Kubelet plugin")
 	logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin "+pluginName), "node", nodeName)
 	ctx = klog.NewContext(ctx, logger)
@@ -444,7 +548,8 @@ func newKubeletPlugin(ctx context.Context, nodeName, pluginName string) *testdri
 		ctx,
 		cdiDir,
 		pluginName,
-		"",
+		clientSet,
+		nodeName,
 		testdriver.FileOperations{},
 		kubeletplugin.PluginSocketPath(endpoint),
 		kubeletplugin.RegistrarSocketPath(path.Join(pluginRegistrationPath, pluginName+"-reg.sock")),
@@ -454,6 +559,11 @@ func newKubeletPlugin(ctx context.Context, nodeName, pluginName string) *testdri
 
 	gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
 
+	ginkgo.DeferCleanup(func(ctx context.Context) {
+		// kubelet should do this eventually, but better make sure.
+		// A separate test checks this explicitly.
+		framework.ExpectNoError(clientSet.ResourceV1alpha2().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: "driverName=" + driverName}))
+	})
 	ginkgo.DeferCleanup(plugin.Stop)
 
 	return plugin
@@ -549,3 +659,27 @@ func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, node
 
 	return pod
 }
+
+func createTestResourceSlice(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName string) {
+	slice := &resourcev1alpha2.ResourceSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: nodeName,
+		},
+		NodeName:   nodeName,
+		DriverName: driverName,
+		ResourceModel: resourcev1alpha2.ResourceModel{
+			NamedResources: &resourcev1alpha2.NamedResourcesResources{},
+		},
+	}
+
+	ginkgo.By(fmt.Sprintf("Creating ResourceSlice %s", nodeName))
+	slice, err := clientSet.ResourceV1alpha2().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{})
+	framework.ExpectNoError(err, "create ResourceSlice")
+	ginkgo.DeferCleanup(func(ctx context.Context) {
+		ginkgo.By(fmt.Sprintf("Deleting ResourceSlice %s", nodeName))
+		err := clientSet.ResourceV1alpha2().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{})
+		if !apierrors.IsNotFound(err) {
+			framework.ExpectNoError(err, "delete ResourceSlice")
+		}
+	})
+}

test/integration/auth/node_test.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/test/integration/framework"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 )
 
 func TestNodeAuthorizer(t *testing.T) {
@@ -113,6 +114,13 @@ func TestNodeAuthorizer(t *testing.T) {
 	if _, err := superuserClient.ResourceV1alpha2().ResourceClaims("ns").Create(context.TODO(), &v1alpha2.ResourceClaim{ObjectMeta: metav1.ObjectMeta{Name: "mytemplatizedresourceclaim"}, Spec: v1alpha2.ResourceClaimSpec{ResourceClassName: "example.com"}}, metav1.CreateOptions{}); err != nil {
 		t.Fatal(err)
 	}
+	model := v1alpha2.ResourceModel{NamedResources: &v1alpha2.NamedResourcesResources{}}
+	if _, err := superuserClient.ResourceV1alpha2().ResourceSlices().Create(context.TODO(), &v1alpha2.ResourceSlice{ObjectMeta: metav1.ObjectMeta{Name: "myslice1"}, NodeName: "node1", DriverName: "dra.example.com", ResourceModel: model}, metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := superuserClient.ResourceV1alpha2().ResourceSlices().Create(context.TODO(), &v1alpha2.ResourceSlice{ObjectMeta: metav1.ObjectMeta{Name: "myslice2"}, NodeName: "node2", DriverName: "dra.example.com", ResourceModel: model}, metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
 
 	pvName := "mypv"
 	if _, err := superuserClientExternal.StorageV1().VolumeAttachments().Create(context.TODO(), &storagev1.VolumeAttachment{
@@ -195,6 +203,15 @@ func TestNodeAuthorizer(t *testing.T) {
 			return err
 		}
 	}
+	deleteResourceSliceCollection := func(client clientset.Interface, nodeName *string) func() error {
+		return func() error {
+			var listOptions metav1.ListOptions
+			if nodeName != nil {
+				listOptions.FieldSelector = "nodeName=" + *nodeName
+			}
+			return client.ResourceV1alpha2().ResourceSlices().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, listOptions)
+		}
+	}
 	addResourceClaimTemplateReference := func(client clientset.Interface) func() error {
 		return func() error {
 			_, err := client.CoreV1().Pods("ns").Patch(context.TODO(), "node2normalpod", types.MergePatchType,
@@ -640,6 +657,32 @@ func TestNodeAuthorizer(t *testing.T) {
 	expectForbidden(t, updateNode1CSINode(csiNode2Client))
 	expectForbidden(t, patchNode1CSINode(csiNode2Client))
 	expectForbidden(t, deleteNode1CSINode(csiNode2Client))
+
+	// Always allowed. Permission to delete specific objects is checked per object.
+	// Beware, this is destructive!
+	expectAllowed(t, deleteResourceSliceCollection(csiNode1Client, ptr.To("node1")))
+
+	// One slice must have been deleted, the other not.
+	slices, err := superuserClient.ResourceV1alpha2().ResourceSlices().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(slices.Items) != 1 {
+		t.Fatalf("unexpected slices: %v", slices.Items)
+	}
+	if slices.Items[0].NodeName != "node2" {
+		t.Fatal("wrong slice deleted")
+	}
+
+	// Superuser can delete.
+	expectAllowed(t, deleteResourceSliceCollection(superuserClient, nil))
+	slices, err = superuserClient.ResourceV1alpha2().ResourceSlices().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(slices.Items) != 0 {
+		t.Fatalf("unexpected slices: %v", slices.Items)
+	}
 }
 
 // expect executes a function a set number of times until it either returns the