From de5742ae83c8d77268a7caf5f3b1f418c4a13a84 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 13 Jun 2024 17:25:39 +0200 Subject: [PATCH] DRA: remove immediate allocation As agreed in https://github.com/kubernetes/enhancements/pull/4709, immediate allocation is one of those features which can be removed because it makes no sense for structured parameters and the justification for classic DRA is weak. --- api/openapi-spec/swagger.json | 4 - ...is__resource.k8s.io__v1alpha3_openapi.json | 4 - pkg/api/testing/defaulting_test.go | 4 - pkg/apis/resource/fuzzer/fuzzer.go | 16 +- pkg/apis/resource/types.go | 26 -- pkg/apis/resource/v1alpha3/defaults.go | 7 - pkg/apis/resource/v1alpha3/defaults_test.go | 75 ---- .../v1alpha3/zz_generated.conversion.go | 2 - .../v1alpha3/zz_generated.defaults.go | 29 -- pkg/apis/resource/validation/validation.go | 5 - .../validation_resourceclaim_test.go | 25 -- .../validation_resourceclaimtemplate_test.go | 24 -- pkg/controller/resourceclaim/controller.go | 19 +- .../resourceclaim/controller_test.go | 76 +---- pkg/generated/openapi/zz_generated.openapi.go | 7 - pkg/printers/internalversion/printers.go | 6 +- .../resourceclaim/storage/storage_test.go | 1 - .../resource/resourceclaim/strategy_test.go | 1 - .../storage/storage_test.go | 1 - .../resourceclaimtemplate/strategy_test.go | 1 - .../dynamicresources/dynamicresources.go | 85 ++--- .../dynamicresources/dynamicresources_test.go | 246 ++++---------- pkg/scheduler/testing/wrappers.go | 6 - .../api/resource/v1alpha3/generated.pb.go | 320 ++++++++---------- .../api/resource/v1alpha3/generated.proto | 5 - .../src/k8s.io/api/resource/v1alpha3/types.go | 26 -- .../v1alpha3/types_swagger_doc_generated.go | 1 - ...esource.k8s.io.v1alpha3.ResourceClaim.json | 3 +- .../resource.k8s.io.v1alpha3.ResourceClaim.pb | Bin 1031 -> 1010 bytes ...esource.k8s.io.v1alpha3.ResourceClaim.yaml | 1 - ...k8s.io.v1alpha3.ResourceClaimTemplate.json | 3 +- ...e.k8s.io.v1alpha3.ResourceClaimTemplate.pb | Bin 861 -> 840 bytes ...k8s.io.v1alpha3.ResourceClaimTemplate.yaml | 1 - .../applyconfigurations/internal/internal.go | 3 - .../resource/v1alpha3/resourceclaimspec.go | 13 - .../controller/controller.go | 47 +-- .../controller/controller_test.go | 200 +++-------- test/e2e/dra/dra.go | 291 +++++----------- test/e2e/dra/test-driver/README.md | 4 +- .../deploy/example/resourceclaim.yaml | 1 - test/integration/etcd/data.go | 4 +- 41 files changed, 395 insertions(+), 1198 deletions(-) delete mode 100644 pkg/apis/resource/v1alpha3/defaults_test.go diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index c68b918fe99..a6f87726213 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -15589,10 +15589,6 @@ "io.k8s.api.resource.v1alpha3.ResourceClaimSpec": { "description": "ResourceClaimSpec defines how a resource is to be allocated.", "properties": { - "allocationMode": { - "description": "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.", - "type": "string" - }, "parametersRef": { "$ref": "#/definitions/io.k8s.api.resource.v1alpha3.ResourceClaimParametersReference", "description": "ParametersRef references a separate object with arbitrary parameters that will be used by the driver when allocating a resource for the claim.\n\nThe object must be in the same namespace as the ResourceClaim." diff --git a/api/openapi-spec/v3/apis__resource.k8s.io__v1alpha3_openapi.json b/api/openapi-spec/v3/apis__resource.k8s.io__v1alpha3_openapi.json index 255c43c70dd..c0eda018d5a 100644 --- a/api/openapi-spec/v3/apis__resource.k8s.io__v1alpha3_openapi.json +++ b/api/openapi-spec/v3/apis__resource.k8s.io__v1alpha3_openapi.json @@ -762,10 +762,6 @@ "io.k8s.api.resource.v1alpha3.ResourceClaimSpec": { "description": "ResourceClaimSpec defines how a resource is to be allocated.", "properties": { - "allocationMode": { - "description": "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.", - "type": "string" - }, "parametersRef": { "allOf": [ { diff --git a/pkg/api/testing/defaulting_test.go b/pkg/api/testing/defaulting_test.go index e19e3376d95..a442b14dfbf 100644 --- a/pkg/api/testing/defaulting_test.go +++ b/pkg/api/testing/defaulting_test.go @@ -135,10 +135,6 @@ func TestDefaulting(t *testing.T) { {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBindingList"}: {}, {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "RoleBinding"}: {}, {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "RoleBindingList"}: {}, - {Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaim"}: {}, - {Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaimList"}: {}, - {Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaimTemplate"}: {}, - {Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaimTemplateList"}: {}, {Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ValidatingAdmissionPolicy"}: {}, {Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ValidatingAdmissionPolicyList"}: {}, {Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ValidatingAdmissionPolicyBinding"}: {}, diff --git a/pkg/apis/resource/fuzzer/fuzzer.go b/pkg/apis/resource/fuzzer/fuzzer.go index 5cf5b5e635b..5f9b5273900 100644 --- a/pkg/apis/resource/fuzzer/fuzzer.go +++ b/pkg/apis/resource/fuzzer/fuzzer.go @@ -17,24 +17,10 @@ limitations under the License. package fuzzer import ( - fuzz "github.com/google/gofuzz" - runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/kubernetes/pkg/apis/resource" ) // Funcs contains the fuzzer functions for the resource group. var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { - return []interface{}{ - func(obj *resource.ResourceClaimSpec, c fuzz.Continue) { - c.FuzzNoCustom(obj) // fuzz self without calling this function again - - // Custom fuzzing for allocation mode: pick one valid mode randomly. - modes := []resource.AllocationMode{ - resource.AllocationModeImmediate, - resource.AllocationModeWaitForFirstConsumer, - } - obj.AllocationMode = modes[c.Rand.Intn(len(modes))] - }, - } + return nil } diff --git a/pkg/apis/resource/types.go b/pkg/apis/resource/types.go index fa3fba7f39e..6a633c1782c 100644 --- a/pkg/apis/resource/types.go +++ b/pkg/apis/resource/types.go @@ -62,34 +62,8 @@ type ResourceClaimSpec struct { // The object must be in the same namespace as the ResourceClaim. // +optional ParametersRef *ResourceClaimParametersReference - - // Allocation can start immediately or when a Pod wants to use the - // resource. "WaitForFirstConsumer" is the default. - // +optional - AllocationMode AllocationMode } -// AllocationMode describes whether a ResourceClaim gets allocated immediately -// when it gets created (AllocationModeImmediate) or whether allocation is -// delayed until it is needed for a Pod -// (AllocationModeWaitForFirstConsumer). Other modes might get added in the -// future. -type AllocationMode string - -const ( - // When a ResourceClaim has AllocationModeWaitForFirstConsumer, allocation is - // delayed until a Pod gets scheduled that needs the ResourceClaim. The - // scheduler will consider all resource requirements of that Pod and - // trigger allocation for a node that fits the Pod. - AllocationModeWaitForFirstConsumer AllocationMode = "WaitForFirstConsumer" - - // When a ResourceClaim has AllocationModeImmediate, allocation starts - // as soon as the ResourceClaim gets created. This is done without - // considering the needs of Pods that will use the ResourceClaim - // because those Pods are not known yet. - AllocationModeImmediate AllocationMode = "Immediate" -) - // ResourceClaimStatus tracks whether the resource has been allocated and what // the resulting attributes are. type ResourceClaimStatus struct { diff --git a/pkg/apis/resource/v1alpha3/defaults.go b/pkg/apis/resource/v1alpha3/defaults.go index 3d3a6c598e5..84851c3dbb2 100644 --- a/pkg/apis/resource/v1alpha3/defaults.go +++ b/pkg/apis/resource/v1alpha3/defaults.go @@ -17,16 +17,9 @@ limitations under the License. package v1alpha3 import ( - "k8s.io/api/resource/v1alpha3" "k8s.io/apimachinery/pkg/runtime" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { return RegisterDefaults(scheme) } - -func SetDefaults_ResourceClaimSpec(obj *v1alpha3.ResourceClaimSpec) { - if obj.AllocationMode == "" { - obj.AllocationMode = v1alpha3.AllocationModeWaitForFirstConsumer - } -} diff --git a/pkg/apis/resource/v1alpha3/defaults_test.go b/pkg/apis/resource/v1alpha3/defaults_test.go deleted file mode 100644 index ad0589a6b0d..00000000000 --- a/pkg/apis/resource/v1alpha3/defaults_test.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha3_test - -import ( - "reflect" - "testing" - - v1alpha3 "k8s.io/api/resource/v1alpha3" - "k8s.io/apimachinery/pkg/runtime" - - // ensure types are installed - "k8s.io/kubernetes/pkg/api/legacyscheme" - _ "k8s.io/kubernetes/pkg/apis/resource/install" -) - -func TestSetDefaultAllocationMode(t *testing.T) { - claim := &v1alpha3.ResourceClaim{} - - // field should be defaulted - defaultMode := v1alpha3.AllocationModeWaitForFirstConsumer - output := roundTrip(t, runtime.Object(claim)).(*v1alpha3.ResourceClaim) - outMode := output.Spec.AllocationMode - if outMode != defaultMode { - t.Errorf("Expected AllocationMode to be defaulted to: %+v, got: %+v", defaultMode, outMode) - } - - // field should not change - nonDefaultMode := v1alpha3.AllocationModeImmediate - claim = &v1alpha3.ResourceClaim{ - Spec: v1alpha3.ResourceClaimSpec{ - AllocationMode: nonDefaultMode, - }, - } - output = roundTrip(t, runtime.Object(claim)).(*v1alpha3.ResourceClaim) - outMode = output.Spec.AllocationMode - if outMode != v1alpha3.AllocationModeImmediate { - t.Errorf("Expected AllocationMode to remain %+v, got: %+v", nonDefaultMode, outMode) - } -} - -func roundTrip(t *testing.T, obj runtime.Object) runtime.Object { - codec := legacyscheme.Codecs.LegacyCodec(v1alpha3.SchemeGroupVersion) - data, err := runtime.Encode(codec, obj) - if err != nil { - t.Errorf("%v\n %#v", err, obj) - return nil - } - obj2, err := runtime.Decode(codec, data) - if err != nil { - t.Errorf("%v\nData: %s\nSource: %#v", err, string(data), obj) - return nil - } - obj3 := reflect.New(reflect.TypeOf(obj).Elem()).Interface().(runtime.Object) - err = legacyscheme.Scheme.Convert(obj2, obj3, nil) - if err != nil { - t.Errorf("%v\nSource: %#v", err, obj2) - return nil - } - return obj3 -} diff --git a/pkg/apis/resource/v1alpha3/zz_generated.conversion.go b/pkg/apis/resource/v1alpha3/zz_generated.conversion.go index 61f339e8eaa..f5d8a633a83 100644 --- a/pkg/apis/resource/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/resource/v1alpha3/zz_generated.conversion.go @@ -1159,7 +1159,6 @@ func Convert_resource_ResourceClaimSchedulingStatus_To_v1alpha3_ResourceClaimSch func autoConvert_v1alpha3_ResourceClaimSpec_To_resource_ResourceClaimSpec(in *v1alpha3.ResourceClaimSpec, out *resource.ResourceClaimSpec, s conversion.Scope) error { out.ResourceClassName = in.ResourceClassName out.ParametersRef = (*resource.ResourceClaimParametersReference)(unsafe.Pointer(in.ParametersRef)) - out.AllocationMode = resource.AllocationMode(in.AllocationMode) return nil } @@ -1171,7 +1170,6 @@ func Convert_v1alpha3_ResourceClaimSpec_To_resource_ResourceClaimSpec(in *v1alph func autoConvert_resource_ResourceClaimSpec_To_v1alpha3_ResourceClaimSpec(in *resource.ResourceClaimSpec, out *v1alpha3.ResourceClaimSpec, s conversion.Scope) error { out.ResourceClassName = in.ResourceClassName out.ParametersRef = (*v1alpha3.ResourceClaimParametersReference)(unsafe.Pointer(in.ParametersRef)) - out.AllocationMode = v1alpha3.AllocationMode(in.AllocationMode) return nil } diff --git a/pkg/apis/resource/v1alpha3/zz_generated.defaults.go b/pkg/apis/resource/v1alpha3/zz_generated.defaults.go index 2ace4bcd3e8..8b3ec1e1c4a 100644 --- a/pkg/apis/resource/v1alpha3/zz_generated.defaults.go +++ b/pkg/apis/resource/v1alpha3/zz_generated.defaults.go @@ -22,7 +22,6 @@ limitations under the License. package v1alpha3 import ( - v1alpha3 "k8s.io/api/resource/v1alpha3" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -30,33 +29,5 @@ import ( // Public to allow building arbitrary schemes. // All generated defaulters are covering - they call all nested defaulters. func RegisterDefaults(scheme *runtime.Scheme) error { - scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaim{}, func(obj interface{}) { SetObjectDefaults_ResourceClaim(obj.(*v1alpha3.ResourceClaim)) }) - scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaimList{}, func(obj interface{}) { SetObjectDefaults_ResourceClaimList(obj.(*v1alpha3.ResourceClaimList)) }) - scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaimTemplate{}, func(obj interface{}) { SetObjectDefaults_ResourceClaimTemplate(obj.(*v1alpha3.ResourceClaimTemplate)) }) - scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaimTemplateList{}, func(obj interface{}) { - SetObjectDefaults_ResourceClaimTemplateList(obj.(*v1alpha3.ResourceClaimTemplateList)) - }) return nil } - -func SetObjectDefaults_ResourceClaim(in *v1alpha3.ResourceClaim) { - SetDefaults_ResourceClaimSpec(&in.Spec) -} - -func SetObjectDefaults_ResourceClaimList(in *v1alpha3.ResourceClaimList) { - for i := range in.Items { - a := &in.Items[i] - SetObjectDefaults_ResourceClaim(a) - } -} - -func SetObjectDefaults_ResourceClaimTemplate(in *v1alpha3.ResourceClaimTemplate) { - SetDefaults_ResourceClaimSpec(&in.Spec.Spec) -} - -func SetObjectDefaults_ResourceClaimTemplateList(in *v1alpha3.ResourceClaimTemplateList) { - for i := range in.Items { - a := &in.Items[i] - SetObjectDefaults_ResourceClaimTemplate(a) - } -} diff --git a/pkg/apis/resource/validation/validation.go b/pkg/apis/resource/validation/validation.go index 7b93aae7db4..39db1d71b66 100644 --- a/pkg/apis/resource/validation/validation.go +++ b/pkg/apis/resource/validation/validation.go @@ -46,14 +46,9 @@ func validateResourceClaimSpec(spec *resource.ResourceClaimSpec, fldPath *field. allErrs = append(allErrs, field.Invalid(fldPath.Child("resourceClassName"), spec.ResourceClassName, msg)) } allErrs = append(allErrs, validateResourceClaimParametersRef(spec.ParametersRef, fldPath.Child("parametersRef"))...) - if !supportedAllocationModes.Has(string(spec.AllocationMode)) { - allErrs = append(allErrs, field.NotSupported(fldPath.Child("allocationMode"), spec.AllocationMode, supportedAllocationModes.List())) - } return allErrs } -var supportedAllocationModes = sets.NewString(string(resource.AllocationModeImmediate), string(resource.AllocationModeWaitForFirstConsumer)) - // It would have been nice to use Go generics to reuse the same validation // function for Kind and Name in both types, but generics cannot be used to // access common fields in structs. diff --git a/pkg/apis/resource/validation/validation_resourceclaim_test.go b/pkg/apis/resource/validation/validation_resourceclaim_test.go index 5c735baef53..3836e0d9f44 100644 --- a/pkg/apis/resource/validation/validation_resourceclaim_test.go +++ b/pkg/apis/resource/validation/validation_resourceclaim_test.go @@ -42,14 +42,11 @@ func testClaim(name, namespace string, spec resource.ResourceClaimSpec) *resourc } func TestValidateClaim(t *testing.T) { - validMode := resource.AllocationModeImmediate - invalidMode := resource.AllocationMode("invalid") goodName := "foo" badName := "!@#$%^" goodNS := "ns" goodClaimSpec := resource.ResourceClaimSpec{ ResourceClassName: goodName, - AllocationMode: validMode, } now := metav1.Now() badValue := "spaces not allowed" @@ -200,14 +197,6 @@ func TestValidateClaim(t *testing.T) { return claim }(), }, - "bad-mode": { - wantFailures: field.ErrorList{field.NotSupported(field.NewPath("spec", "allocationMode"), invalidMode, supportedAllocationModes.List())}, - claim: func() *resource.ResourceClaim { - claim := testClaim(goodName, goodNS, goodClaimSpec) - claim.Spec.AllocationMode = invalidMode - return claim - }(), - }, "good-parameters": { claim: func() *resource.ResourceClaim { claim := testClaim(goodName, goodNS, goodClaimSpec) @@ -279,7 +268,6 @@ func TestValidateClaimUpdate(t *testing.T) { } validClaim := testClaim("foo", "ns", resource.ResourceClaimSpec{ ResourceClassName: name, - AllocationMode: resource.AllocationModeImmediate, ParametersRef: parameters, }) @@ -316,18 +304,6 @@ func TestValidateClaimUpdate(t *testing.T) { return claim }, }, - "invalid-update-mode": { - wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), func() resource.ResourceClaimSpec { - spec := validClaim.Spec.DeepCopy() - spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer - return *spec - }(), "field is immutable")}, - oldClaim: validClaim, - update: func(claim *resource.ResourceClaim) *resource.ResourceClaim { - claim.Spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer - return claim - }, - }, } for name, scenario := range scenarios { @@ -343,7 +319,6 @@ func TestValidateClaimStatusUpdate(t *testing.T) { invalidName := "!@#$%^" validClaim := testClaim("foo", "ns", resource.ResourceClaimSpec{ ResourceClassName: "valid", - AllocationMode: resource.AllocationModeImmediate, }) validAllocatedClaim := validClaim.DeepCopy() diff --git a/pkg/apis/resource/validation/validation_resourceclaimtemplate_test.go b/pkg/apis/resource/validation/validation_resourceclaimtemplate_test.go index 3f6c5504147..ba51420c268 100644 --- a/pkg/apis/resource/validation/validation_resourceclaimtemplate_test.go +++ b/pkg/apis/resource/validation/validation_resourceclaimtemplate_test.go @@ -40,14 +40,11 @@ func testClaimTemplate(name, namespace string, spec resource.ResourceClaimSpec) } func TestValidateClaimTemplate(t *testing.T) { - validMode := resource.AllocationModeImmediate - invalidMode := resource.AllocationMode("invalid") goodName := "foo" badName := "!@#$%^" goodNS := "ns" goodClaimSpec := resource.ResourceClaimSpec{ ResourceClassName: goodName, - AllocationMode: validMode, } now := metav1.Now() badValue := "spaces not allowed" @@ -198,14 +195,6 @@ func TestValidateClaimTemplate(t *testing.T) { return template }(), }, - "bad-mode": { - wantFailures: field.ErrorList{field.NotSupported(field.NewPath("spec", "spec", "allocationMode"), invalidMode, supportedAllocationModes.List())}, - template: func() *resource.ResourceClaimTemplate { - template := testClaimTemplate(goodName, goodNS, goodClaimSpec) - template.Spec.Spec.AllocationMode = invalidMode - return template - }(), - }, "good-parameters": { template: func() *resource.ResourceClaimTemplate { template := testClaimTemplate(goodName, goodNS, goodClaimSpec) @@ -277,7 +266,6 @@ func TestValidateClaimTemplateUpdate(t *testing.T) { } validClaimTemplate := testClaimTemplate("foo", "ns", resource.ResourceClaimSpec{ ResourceClassName: name, - AllocationMode: resource.AllocationModeImmediate, ParametersRef: parameters, }) @@ -314,18 +302,6 @@ func TestValidateClaimTemplateUpdate(t *testing.T) { return template }, }, - "invalid-update-mode": { - wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), func() resource.ResourceClaimTemplateSpec { - spec := validClaimTemplate.Spec.DeepCopy() - spec.Spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer - return *spec - }(), "field is immutable")}, - oldClaimTemplate: validClaimTemplate, - update: func(template *resource.ResourceClaimTemplate) *resource.ResourceClaimTemplate { - template.Spec.Spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer - return template - }, - }, } for name, scenario := range scenarios { diff --git a/pkg/controller/resourceclaim/controller.go b/pkg/controller/resourceclaim/controller.go index bb14fe062b9..596113b17d2 100644 --- a/pkg/controller/resourceclaim/controller.go +++ b/pkg/controller/resourceclaim/controller.go @@ -329,8 +329,7 @@ func (ec *Controller) podNeedsWork(pod *v1.Pod) (bool, string) { // - a user created a pod with spec.nodeName set, perhaps for testing // - some scheduler was used which is unaware of DRA // - DRA was not enabled in kube-scheduler (version skew, configuration) - if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer && - claim.Status.Allocation == nil { + if claim.Status.Allocation == nil { scheduling, err := ec.podSchedulingLister.PodSchedulingContexts(pod.Namespace).Get(pod.Name) if apierrors.IsNotFound(err) { return true, "need to create PodSchedulingContext for scheduled pod" @@ -533,8 +532,7 @@ func (ec *Controller) syncPod(ctx context.Context, namespace, name string) error return err } } - if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer && - claim.Status.Allocation == nil { + if claim.Status.Allocation == nil { logger.V(5).Info("create PodSchedulingContext because claim needs to be allocated", "resourceClaim", klog.KObj(claim)) return ec.ensurePodSchedulingContext(ctx, pod) } @@ -864,19 +862,14 @@ func (ec *Controller) syncClaim(ctx context.Context, namespace, name string) err // for such claims and not checking for them keeps this code simpler. if len(valid) == 0 { if builtinControllerFinalizer >= 0 { - if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer || - claim.DeletionTimestamp != nil { - // Allocated by scheduler with structured parameters. We can "deallocate" - // by clearing the allocation. - claim.Status.Allocation = nil - } - } else if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer { + // Allocated by scheduler with structured parameters. We can "deallocate" + // by clearing the allocation. + claim.Status.Allocation = nil + } else { // DRA driver controller in the control plane // needs to do the deallocation. claim.Status.DeallocationRequested = true } - // In all other cases, we keep the claim allocated, in particular for immediate allocation - // with a control plane controller. } claim, err := ec.kubeClient.ResourceV1alpha3().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) diff --git a/pkg/controller/resourceclaim/controller_test.go b/pkg/controller/resourceclaim/controller_test.go index 161cd5fd388..94c88653558 100644 --- a/pkg/controller/resourceclaim/controller_test.go +++ b/pkg/controller/resourceclaim/controller_test.go @@ -281,7 +281,7 @@ func TestSyncHandler(t *testing.T) { expectedMetrics: expectedMetrics{0, 0}, }, { - name: "clear-reserved-delayed-allocation", + name: "clear-reserved", pods: []*v1.Pod{}, key: claimKey(testClaimReserved), claims: []*resourceapi.ResourceClaim{testClaimReserved}, @@ -293,7 +293,7 @@ func TestSyncHandler(t *testing.T) { expectedMetrics: expectedMetrics{0, 0}, }, { - name: "clear-reserved-delayed-allocation-structured", + name: "clear-reserved-structured", pods: []*v1.Pod{}, key: claimKey(testClaimReserved), claims: []*resourceapi.ResourceClaim{structuredParameters(testClaimReserved)}, @@ -306,7 +306,7 @@ func TestSyncHandler(t *testing.T) { expectedMetrics: expectedMetrics{0, 0}, }, { - name: "dont-clear-reserved-delayed-allocation-structured", + name: "dont-clear-reserved-structured", pods: []*v1.Pod{testPodWithResource}, key: claimKey(testClaimReserved), claims: func() []*resourceapi.ResourceClaim { @@ -318,50 +318,16 @@ func TestSyncHandler(t *testing.T) { expectedMetrics: expectedMetrics{0, 0}, }, { - name: "clear-reserved-immediate-allocation", - pods: []*v1.Pod{}, - key: claimKey(testClaimReserved), - claims: func() []*resourceapi.ResourceClaim { - claim := testClaimReserved.DeepCopy() - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate - return []*resourceapi.ResourceClaim{claim} - }(), - expectedClaims: func() []resourceapi.ResourceClaim { - claim := testClaimAllocated.DeepCopy() - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate - return []resourceapi.ResourceClaim{*claim} - }(), - expectedMetrics: expectedMetrics{0, 0}, - }, - { - name: "clear-reserved-immediate-allocation-structured", + name: "clear-reserved-structured-deleted", pods: []*v1.Pod{}, key: claimKey(testClaimReserved), claims: func() []*resourceapi.ResourceClaim { claim := structuredParameters(testClaimReserved.DeepCopy()) - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate - return []*resourceapi.ResourceClaim{claim} - }(), - expectedClaims: func() []resourceapi.ResourceClaim { - claim := structuredParameters(testClaimAllocated.DeepCopy()) - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate - return []resourceapi.ResourceClaim{*claim} - }(), - expectedMetrics: expectedMetrics{0, 0}, - }, - { - name: "clear-reserved-immediate-allocation-structured-deleted", - pods: []*v1.Pod{}, - key: claimKey(testClaimReserved), - claims: func() []*resourceapi.ResourceClaim { - claim := structuredParameters(testClaimReserved.DeepCopy()) - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate claim.DeletionTimestamp = &metav1.Time{} return []*resourceapi.ResourceClaim{claim} }(), expectedClaims: func() []resourceapi.ResourceClaim { claim := structuredParameters(testClaimAllocated.DeepCopy()) - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate claim.DeletionTimestamp = &metav1.Time{} claim.Finalizers = []string{} claim.Status.Allocation = nil @@ -370,18 +336,16 @@ func TestSyncHandler(t *testing.T) { expectedMetrics: expectedMetrics{0, 0}, }, { - name: "immediate-allocation-structured-deleted", + name: "structured-deleted", pods: []*v1.Pod{}, key: claimKey(testClaimReserved), claims: func() []*resourceapi.ResourceClaim { claim := structuredParameters(testClaimAllocated.DeepCopy()) - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate claim.DeletionTimestamp = &metav1.Time{} return []*resourceapi.ResourceClaim{claim} }(), expectedClaims: func() []resourceapi.ResourceClaim { claim := structuredParameters(testClaimAllocated.DeepCopy()) - claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate claim.DeletionTimestamp = &metav1.Time{} claim.Finalizers = []string{} claim.Status.Allocation = nil @@ -390,7 +354,7 @@ func TestSyncHandler(t *testing.T) { expectedMetrics: expectedMetrics{0, 0}, }, { - name: "clear-reserved-when-done-delayed-allocation", + name: "clear-reserved-when-done", pods: func() []*v1.Pod { pods := []*v1.Pod{testPodWithResource.DeepCopy()} pods[0].Status.Phase = v1.PodSucceeded @@ -410,28 +374,6 @@ func TestSyncHandler(t *testing.T) { }(), expectedMetrics: expectedMetrics{0, 0}, }, - { - name: "clear-reserved-when-done-immediate-allocation", - pods: func() []*v1.Pod { - pods := []*v1.Pod{testPodWithResource.DeepCopy()} - pods[0].Status.Phase = v1.PodSucceeded - return pods - }(), - key: claimKey(testClaimReserved), - claims: func() []*resourceapi.ResourceClaim { - claims := []*resourceapi.ResourceClaim{testClaimReserved.DeepCopy()} - claims[0].OwnerReferences = nil - claims[0].Spec.AllocationMode = resourceapi.AllocationModeImmediate - return claims - }(), - expectedClaims: func() []resourceapi.ResourceClaim { - claims := []resourceapi.ResourceClaim{*testClaimAllocated.DeepCopy()} - claims[0].OwnerReferences = nil - claims[0].Spec.AllocationMode = resourceapi.AllocationModeImmediate - return claims - }(), - expectedMetrics: expectedMetrics{0, 0}, - }, { name: "remove-reserved", pods: []*v1.Pod{testPod}, @@ -587,7 +529,6 @@ func makeClaim(name, namespace, classname string, owner *metav1.OwnerReference) ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: resourceapi.ResourceClaimSpec{ ResourceClassName: classname, - AllocationMode: resourceapi.AllocationModeWaitForFirstConsumer, }, } if owner != nil { @@ -607,7 +548,6 @@ func makeGeneratedClaim(podClaimName, generateName, namespace, classname string, }, Spec: resourceapi.ResourceClaimSpec{ ResourceClassName: classname, - AllocationMode: resourceapi.AllocationModeWaitForFirstConsumer, }, } if owner != nil { @@ -709,10 +649,6 @@ func normalizeClaims(claims []resourceapi.ResourceClaim) []resourceapi.ResourceC if len(claims[i].Status.ReservedFor) == 0 { claims[i].Status.ReservedFor = nil } - if claims[i].Spec.AllocationMode == "" { - // This emulates defaulting. - claims[i].Spec.AllocationMode = resourceapi.AllocationModeWaitForFirstConsumer - } } return claims } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 2042532130b..9628cdfef78 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -46572,13 +46572,6 @@ func schema_k8sio_api_resource_v1alpha3_ResourceClaimSpec(ref common.ReferenceCa Ref: ref("k8s.io/api/resource/v1alpha3.ResourceClaimParametersReference"), }, }, - "allocationMode": { - SchemaProps: spec.SchemaProps{ - Description: "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.", - Type: []string{"string"}, - Format: "", - }, - }, }, Required: []string{"resourceClassName"}, }, diff --git a/pkg/printers/internalversion/printers.go b/pkg/printers/internalversion/printers.go index 0b167081b71..ddd0fc83ea7 100644 --- a/pkg/printers/internalversion/printers.go +++ b/pkg/printers/internalversion/printers.go @@ -634,7 +634,6 @@ func AddHandlers(h printers.PrintHandler) { resourceClaimColumnDefinitions := []metav1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, {Name: "ResourceClassName", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["resourceClassName"]}, - {Name: "AllocationMode", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["allocationMode"]}, {Name: "State", Type: "string", Description: "A summary of the current state (allocated, pending, reserved, etc.)."}, {Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]}, } @@ -644,7 +643,6 @@ func AddHandlers(h printers.PrintHandler) { resourceClaimTemplateColumnDefinitions := []metav1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, {Name: "ResourceClassName", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["resourceClassName"]}, - {Name: "AllocationMode", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["allocationMode"]}, {Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]}, } _ = h.TableHandler(resourceClaimTemplateColumnDefinitions, printResourceClaimTemplate) @@ -3006,7 +3004,7 @@ func printResourceClaim(obj *resource.ResourceClaim, options printers.GenerateOp row := metav1.TableRow{ Object: runtime.RawExtension{Object: obj}, } - row.Cells = append(row.Cells, obj.Name, obj.Spec.ResourceClassName, string(obj.Spec.AllocationMode), resourceClaimState(obj), translateTimestampSince(obj.CreationTimestamp)) + row.Cells = append(row.Cells, obj.Name, obj.Spec.ResourceClassName, resourceClaimState(obj), translateTimestampSince(obj.CreationTimestamp)) return []metav1.TableRow{row}, nil } @@ -3047,7 +3045,7 @@ func printResourceClaimTemplate(obj *resource.ResourceClaimTemplate, options pri row := metav1.TableRow{ Object: runtime.RawExtension{Object: obj}, } - row.Cells = append(row.Cells, obj.Name, obj.Spec.Spec.ResourceClassName, string(obj.Spec.Spec.AllocationMode), translateTimestampSince(obj.CreationTimestamp)) + row.Cells = append(row.Cells, obj.Name, obj.Spec.Spec.ResourceClassName, translateTimestampSince(obj.CreationTimestamp)) return []metav1.TableRow{row}, nil } diff --git a/pkg/registry/resource/resourceclaim/storage/storage_test.go b/pkg/registry/resource/resourceclaim/storage/storage_test.go index 613d3c80467..114c5f36b91 100644 --- a/pkg/registry/resource/resourceclaim/storage/storage_test.go +++ b/pkg/registry/resource/resourceclaim/storage/storage_test.go @@ -58,7 +58,6 @@ func validNewClaim(name, ns string) *resource.ResourceClaim { }, Spec: resource.ResourceClaimSpec{ ResourceClassName: "example", - AllocationMode: resource.AllocationModeImmediate, }, Status: resource.ResourceClaimStatus{}, } diff --git a/pkg/registry/resource/resourceclaim/strategy_test.go b/pkg/registry/resource/resourceclaim/strategy_test.go index d4091893b5b..e99439234b0 100644 --- a/pkg/registry/resource/resourceclaim/strategy_test.go +++ b/pkg/registry/resource/resourceclaim/strategy_test.go @@ -31,7 +31,6 @@ var resourceClaim = &resource.ResourceClaim{ }, Spec: resource.ResourceClaimSpec{ ResourceClassName: "valid-class", - AllocationMode: resource.AllocationModeImmediate, }, } diff --git a/pkg/registry/resource/resourceclaimtemplate/storage/storage_test.go b/pkg/registry/resource/resourceclaimtemplate/storage/storage_test.go index 642938b3bc2..332cfc2e851 100644 --- a/pkg/registry/resource/resourceclaimtemplate/storage/storage_test.go +++ b/pkg/registry/resource/resourceclaimtemplate/storage/storage_test.go @@ -55,7 +55,6 @@ func validNewClaimTemplate(name string) *resource.ResourceClaimTemplate { Spec: resource.ResourceClaimTemplateSpec{ Spec: resource.ResourceClaimSpec{ ResourceClassName: "valid-class", - AllocationMode: resource.AllocationModeImmediate, }, }, } diff --git a/pkg/registry/resource/resourceclaimtemplate/strategy_test.go b/pkg/registry/resource/resourceclaimtemplate/strategy_test.go index 47e25375321..1eb8ad05c77 100644 --- a/pkg/registry/resource/resourceclaimtemplate/strategy_test.go +++ b/pkg/registry/resource/resourceclaimtemplate/strategy_test.go @@ -32,7 +32,6 @@ var resourceClaimTemplate = &resource.ResourceClaimTemplate{ Spec: resource.ResourceClaimTemplateSpec{ Spec: resource.ResourceClaimSpec{ ResourceClassName: "valid-class", - AllocationMode: resource.AllocationModeImmediate, }, }, } diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go index fe368184e78..0786776acc3 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go @@ -713,8 +713,7 @@ func (pl *dynamicResources) isSchedulableAfterPodSchedulingContextChange(logger // we allow backoff. pendingDelayedClaims := 0 if err := pl.foreachPodResourceClaim(pod, func(podResourceName string, claim *resourceapi.ResourceClaim) { - if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer && - claim.Status.Allocation == nil && + if claim.Status.Allocation == nil && !podSchedulingHasClaimInfo(podScheduling, podResourceName) { pendingDelayedClaims++ } @@ -970,9 +969,6 @@ func (pl *dynamicResources) PreFilter(ctx context.Context, state *framework.Cycl } s.informationsForClaim[index].controller = controller needResourceInformation = true - } else if claim.Spec.AllocationMode == resourceapi.AllocationModeImmediate { - // This will get resolved by the resource driver. - return nil, statusUnschedulable(logger, "unallocated immediate resourceclaim", "pod", klog.KObj(pod), "resourceclaim", klog.KObj(claim)) } } } @@ -1161,74 +1157,63 @@ func (pl *dynamicResources) Filter(ctx context.Context, cs *framework.CycleState var unavailableClaims []int for index, claim := range state.claims { logger.V(10).Info("filtering based on resource claims of the pod", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) - switch { - case claim.Status.Allocation != nil: + + if claim.Status.Allocation != nil { if nodeSelector := state.informationsForClaim[index].availableOnNode; nodeSelector != nil { if !nodeSelector.Match(node) { logger.V(5).Info("AvailableOnNodes does not match", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) unavailableClaims = append(unavailableClaims, index) } } - case claim.Status.DeallocationRequested: + continue + } + + if claim.Status.DeallocationRequested { // We shouldn't get here. PreFilter already checked this. return statusUnschedulable(logger, "resourceclaim must be reallocated", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) - case claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer || - state.informationsForClaim[index].structuredParameters: - if selector := state.informationsForClaim[index].availableOnNode; selector != nil { - if matches := selector.Match(node); !matches { - return statusUnschedulable(logger, "excluded by resource class node filter", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclassName", claim.Spec.ResourceClassName) - } + } + + if selector := state.informationsForClaim[index].availableOnNode; selector != nil { + if matches := selector.Match(node); !matches { + return statusUnschedulable(logger, "excluded by resource class node filter", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclassName", claim.Spec.ResourceClassName) } - // Can the builtin controller tell us whether the node is suitable? - if state.informationsForClaim[index].structuredParameters { - suitable, err := state.informationsForClaim[index].controller.nodeIsSuitable(ctx, node.Name, state.resources) - if err != nil { - // An error indicates that something wasn't configured correctly, for example - // writing a CEL expression which doesn't handle a map lookup error. Normally - // this should never fail. We could return an error here, but then the pod - // would get retried. Instead we ignore the node. - return statusUnschedulable(logger, fmt.Sprintf("checking structured parameters failed: %v", err), "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) - } - if !suitable { - return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) - } - } else { - if status := state.informationsForClaim[index].status; status != nil { - for _, unsuitableNode := range status.UnsuitableNodes { - if node.Name == unsuitableNode { - return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim), "unsuitablenodes", status.UnsuitableNodes) - } + } + // Can the builtin controller tell us whether the node is suitable? + if state.informationsForClaim[index].structuredParameters { + suitable, err := state.informationsForClaim[index].controller.nodeIsSuitable(ctx, node.Name, state.resources) + if err != nil { + // An error indicates that something wasn't configured correctly, for example + // writing a CEL expression which doesn't handle a map lookup error. Normally + // this should never fail. We could return an error here, but then the pod + // would get retried. Instead we ignore the node. + return statusUnschedulable(logger, fmt.Sprintf("checking structured parameters failed: %v", err), "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) + } + if !suitable { + return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) + } + } else { + if status := state.informationsForClaim[index].status; status != nil { + for _, unsuitableNode := range status.UnsuitableNodes { + if node.Name == unsuitableNode { + return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim), "unsuitablenodes", status.UnsuitableNodes) } } } - default: - // This claim should have been handled above. - // Immediate allocation with control plane controller - // was already checked for in PreFilter. - return statusError(logger, fmt.Errorf("internal error, unexpected allocation mode %v", claim.Spec.AllocationMode)) } } if len(unavailableClaims) > 0 { + // Remember all unavailable claims. This might be observed + // concurrently, so we have to lock the state before writing. state.mutex.Lock() defer state.mutex.Unlock() + if state.unavailableClaims == nil { state.unavailableClaims = sets.New[int]() } for _, index := range unavailableClaims { - claim := state.claims[index] - // Deallocation makes more sense for claims with - // delayed allocation. Claims with immediate allocation - // would just get allocated again for a random node, - // which is unlikely to help the pod. - // - // Claims with builtin controller are handled like - // claims with delayed allocation. - if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer || - state.informationsForClaim[index].controller != nil { - state.unavailableClaims.Insert(index) - } + state.unavailableClaims.Insert(index) } return statusUnschedulable(logger, "resourceclaim not available on the node", "pod", klog.KObj(pod)) } diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go index 7b7c085c925..5e0221d6b48 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go @@ -165,53 +165,39 @@ var ( Namespace(namespace). ResourceClassName(className). Obj() - pendingImmediateClaim = st.FromResourceClaim(claim). - AllocationMode(resourceapi.AllocationModeImmediate). - Obj() - structuredAllocatedImmediateClaim = st.FromResourceClaim(pendingImmediateClaim). - Allocation("some-driver", &resourceapi.AllocationResult{}). - Structured("worker", "instance-1"). - Obj() - pendingDelayedClaim = st.FromResourceClaim(claim). - OwnerReference(podName, podUID, podKind). - AllocationMode(resourceapi.AllocationModeWaitForFirstConsumer). - Obj() - pendingDelayedClaim2 = st.FromResourceClaim(pendingDelayedClaim). - Name(claimName2). - Obj() - deallocatingClaim = st.FromResourceClaim(pendingImmediateClaim). + pendingClaim = st.FromResourceClaim(claim). + OwnerReference(podName, podUID, podKind). + Obj() + pendingClaim2 = st.FromResourceClaim(pendingClaim). + Name(claimName2). + Obj() + deallocatingClaim = st.FromResourceClaim(pendingClaim). Allocation("some-driver", &resourceapi.AllocationResult{}). DeallocationRequested(true). Obj() - inUseClaim = st.FromResourceClaim(pendingImmediateClaim). + inUseClaim = st.FromResourceClaim(pendingClaim). Allocation("some-driver", &resourceapi.AllocationResult{}). ReservedForPod(podName, types.UID(podUID)). Obj() structuredInUseClaim = st.FromResourceClaim(inUseClaim). Structured("worker", "instance-1"). Obj() - allocatedClaim = st.FromResourceClaim(pendingDelayedClaim). + allocatedClaim = st.FromResourceClaim(pendingClaim). Allocation("some-driver", &resourceapi.AllocationResult{}). Obj() - pendingDelayedClaimWithParams = st.FromResourceClaim(pendingDelayedClaim).ParametersRef(claimName).Obj() + pendingClaimWithParams = st.FromResourceClaim(pendingClaim).ParametersRef(claimName).Obj() structuredAllocatedClaim = st.FromResourceClaim(allocatedClaim).Structured("worker", "instance-1").Obj() structuredAllocatedClaimWithParams = st.FromResourceClaim(structuredAllocatedClaim).ParametersRef(claimName).Obj() otherStructuredAllocatedClaim = st.FromResourceClaim(structuredAllocatedClaim).Name(structuredAllocatedClaim.Name + "-other").Obj() - allocatedDelayedClaimWithWrongTopology = st.FromResourceClaim(allocatedClaim). - Allocation("some-driver", &resourceapi.AllocationResult{AvailableOnNodes: st.MakeNodeSelector().In("no-such-label", []string{"no-such-value"}).Obj()}). - Obj() - structuredAllocatedDelayedClaimWithWrongTopology = st.FromResourceClaim(allocatedDelayedClaimWithWrongTopology). - Structured("worker-2", "instance-1"). - Obj() - allocatedImmediateClaimWithWrongTopology = st.FromResourceClaim(allocatedDelayedClaimWithWrongTopology). - AllocationMode(resourceapi.AllocationModeImmediate). + allocatedClaimWithWrongTopology = st.FromResourceClaim(allocatedClaim). + Allocation("some-driver", &resourceapi.AllocationResult{AvailableOnNodes: st.MakeNodeSelector().In("no-such-label", []string{"no-such-value"}).Obj()}). + Obj() + structuredAllocatedClaimWithWrongTopology = st.FromResourceClaim(allocatedClaimWithWrongTopology). + Structured("worker-2", "instance-1"). Obj() - structuredAllocatedImmediateClaimWithWrongTopology = st.FromResourceClaim(allocatedImmediateClaimWithWrongTopology). - Structured("worker-2", "instance-1"). - Obj() allocatedClaimWithGoodTopology = st.FromResourceClaim(allocatedClaim). Allocation("some-driver", &resourceapi.AllocationResult{AvailableOnNodes: st.MakeNodeSelector().In("kubernetes.io/hostname", []string{"worker"}).Obj()}). Obj() @@ -483,22 +469,9 @@ func TestPlugin(t *testing.T) { }, }, }, - "waiting-for-immediate-allocation": { + "structured-no-resources": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingImmediateClaim}, - classes: []*resourceapi.ResourceClass{resourceClass}, - want: want{ - prefilter: result{ - status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `unallocated immediate resourceclaim`), - }, - postfilter: result{ - status: framework.NewStatus(framework.Unschedulable, `no new claims to deallocate`), - }, - }, - }, - "immediate-allocation-structured-no-resources": { - pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingImmediateClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, classes: []*resourceapi.ResourceClass{structuredResourceClass}, want: want{ filter: perNodeResult{ @@ -511,51 +484,9 @@ func TestPlugin(t *testing.T) { }, }, }, - "immediate-allocation-structured-with-resources": { + "structured-with-resources": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingImmediateClaim}, - classes: []*resourceapi.ResourceClass{structuredResourceClass}, - objs: []apiruntime.Object{workerNodeSlice}, - want: want{ - reserve: result{ - inFlightClaim: structuredAllocatedImmediateClaim, - }, - prebind: result{ - assumedClaim: reserve(structuredAllocatedImmediateClaim, podWithClaimName), - changes: change{ - claim: func(claim *resourceapi.ResourceClaim) *resourceapi.ResourceClaim { - if claim.Name == claimName { - claim = claim.DeepCopy() - claim.Finalizers = structuredAllocatedImmediateClaim.Finalizers - claim.Status = structuredInUseClaim.Status - } - return claim - }, - }, - }, - postbind: result{ - assumedClaim: reserve(structuredAllocatedImmediateClaim, podWithClaimName), - }, - }, - }, - "delayed-allocation-structured-no-resources": { - pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, - classes: []*resourceapi.ResourceClass{structuredResourceClass}, - want: want{ - filter: perNodeResult{ - workerNode.Name: { - status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `resourceclaim cannot be allocated for the node (unsuitable)`), - }, - }, - postfilter: result{ - status: framework.NewStatus(framework.Unschedulable, `still not schedulable`), - }, - }, - }, - "delayed-allocation-structured-with-resources": { - pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, classes: []*resourceapi.ResourceClass{structuredResourceClass}, objs: []apiruntime.Object{workerNodeSlice}, want: want{ @@ -580,12 +511,12 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-structured-with-resources-has-finalizer": { + "structured-with-resources-has-finalizer": { // As before. but the finalizer is already set. Could happen if // the scheduler got interrupted. pod: podWithClaimName, claims: func() []*resourceapi.ResourceClaim { - claim := pendingDelayedClaim.DeepCopy() + claim := pendingClaim.DeepCopy() claim.Finalizers = structuredAllocatedClaim.Finalizers return []*resourceapi.ResourceClaim{claim} }(), @@ -612,12 +543,12 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-structured-with-resources-finalizer-gets-removed": { + "structured-with-resources-finalizer-gets-removed": { // As before. but the finalizer is already set. Then it gets // removed before the scheduler reaches PreBind. pod: podWithClaimName, claims: func() []*resourceapi.ResourceClaim { - claim := pendingDelayedClaim.DeepCopy() + claim := pendingClaim.DeepCopy() claim.Finalizers = structuredAllocatedClaim.Finalizers return []*resourceapi.ResourceClaim{claim} }(), @@ -653,11 +584,11 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-structured-with-resources-finalizer-gets-added": { + "structured-with-resources-finalizer-gets-added": { // No finalizer initially, then it gets added before // the scheduler reaches PreBind. Shouldn't happen? pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, classes: []*resourceapi.ResourceClass{structuredResourceClass}, objs: []apiruntime.Object{workerNodeSlice}, prepare: prepare{ @@ -689,9 +620,9 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-structured-skip-bind": { + "structured-skip-bind": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, classes: []*resourceapi.ResourceClass{structuredResourceClass}, objs: []apiruntime.Object{workerNodeSlice}, want: want{ @@ -701,9 +632,9 @@ func TestPlugin(t *testing.T) { unreserveBeforePreBind: &result{}, }, }, - "delayed-allocation-structured-exhausted-resources": { + "structured-exhausted-resources": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim, otherStructuredAllocatedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim, otherStructuredAllocatedClaim}, classes: []*resourceapi.ResourceClass{structuredResourceClass}, objs: []apiruntime.Object{workerNodeSlice}, want: want{ @@ -720,7 +651,7 @@ func TestPlugin(t *testing.T) { "with-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams}, + claims: []*resourceapi.ResourceClaim{pendingClaimWithParams}, classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams}, objs: []apiruntime.Object{claimParameters, classParameters, workerNodeSlice}, want: want{ @@ -748,7 +679,7 @@ func TestPlugin(t *testing.T) { "with-translated-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)}, + claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)}, classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)}, objs: []apiruntime.Object{claimParameters, claimParametersOtherNamespace /* must be ignored */, classParameters, workerNodeSlice}, want: want{ @@ -776,7 +707,7 @@ func TestPlugin(t *testing.T) { "missing-class-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams}, + claims: []*resourceapi.ResourceClaim{pendingClaimWithParams}, classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams}, objs: []apiruntime.Object{claimParameters, workerNodeSlice}, want: want{ @@ -791,7 +722,7 @@ func TestPlugin(t *testing.T) { "missing-claim-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams}, + claims: []*resourceapi.ResourceClaim{pendingClaimWithParams}, classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams}, objs: []apiruntime.Object{classParameters, workerNodeSlice}, want: want{ @@ -806,7 +737,7 @@ func TestPlugin(t *testing.T) { "missing-translated-class-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)}, + claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)}, classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)}, objs: []apiruntime.Object{claimParameters, workerNodeSlice}, want: want{ @@ -821,7 +752,7 @@ func TestPlugin(t *testing.T) { "missing-translated-claim-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)}, + claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)}, classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)}, objs: []apiruntime.Object{classParameters, workerNodeSlice}, want: want{ @@ -836,7 +767,7 @@ func TestPlugin(t *testing.T) { "too-many-translated-class-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)}, + claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)}, classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)}, objs: []apiruntime.Object{claimParameters, classParameters, st.FromClassParameters(classParameters).Name("other").Obj() /* too many */, workerNodeSlice}, want: want{ @@ -851,7 +782,7 @@ func TestPlugin(t *testing.T) { "too-many-translated-claim-parameters": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)}, + claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)}, classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)}, objs: []apiruntime.Object{claimParameters, st.FromClaimParameters(claimParameters).Name("other").Obj() /* too many */, classParameters, workerNodeSlice}, want: want{ @@ -866,7 +797,7 @@ func TestPlugin(t *testing.T) { "claim-parameters-CEL-runtime-error": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams}, + claims: []*resourceapi.ResourceClaim{pendingClaimWithParams}, classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams}, objs: []apiruntime.Object{breakCELInClaimParameters(claimParameters), classParameters, workerNodeSlice}, want: want{ @@ -883,7 +814,7 @@ func TestPlugin(t *testing.T) { "class-parameters-CEL-runtime-error": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams}, + claims: []*resourceapi.ResourceClaim{pendingClaimWithParams}, classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams}, objs: []apiruntime.Object{claimParameters, breakCELInClassParameters(classParameters), workerNodeSlice}, want: want{ @@ -910,9 +841,9 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-missing-class": { + "missing-class": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, want: want{ prefilter: result{ status: framework.NewStatus(framework.UnschedulableAndUnresolvable, fmt.Sprintf("resource class %s does not exist", className)), @@ -922,11 +853,11 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-scheduling-select-immediately": { + "scheduling-select-immediately": { // Create the PodSchedulingContext object, ask for information // and select a node. pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, classes: []*resourceapi.ResourceClass{resourceClass}, want: want{ prebind: result{ @@ -935,12 +866,12 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-scheduling-ask": { + "scheduling-ask": { // Create the PodSchedulingContext object, ask for // information, but do not select a node because // there are multiple claims. pod: podWithTwoClaimNames, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim, pendingDelayedClaim2}, + claims: []*resourceapi.ResourceClaim{pendingClaim, pendingClaim2}, classes: []*resourceapi.ResourceClass{resourceClass}, want: want{ prebind: result{ @@ -949,11 +880,11 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-scheduling-finish": { + "scheduling-finish": { // Use the populated PodSchedulingContext object to select a // node. pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, schedulings: []*resourceapi.PodSchedulingContext{schedulingInfo}, classes: []*resourceapi.ResourceClass{resourceClass}, want: want{ @@ -969,11 +900,11 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-scheduling-finish-concurrent-label-update": { + "scheduling-finish-concurrent-label-update": { // Use the populated PodSchedulingContext object to select a // node. pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, schedulings: []*resourceapi.PodSchedulingContext{schedulingInfo}, classes: []*resourceapi.ResourceClass{resourceClass}, prepare: prepare{ @@ -994,7 +925,7 @@ func TestPlugin(t *testing.T) { }, }, }, - "delayed-allocation-scheduling-completed": { + "scheduling-completed": { // Remove PodSchedulingContext object once the pod is scheduled. pod: podWithClaimName, claims: []*resourceapi.ResourceClaim{allocatedClaim}, @@ -1031,11 +962,11 @@ func TestPlugin(t *testing.T) { }, }, }, - "wrong-topology-delayed-allocation": { + "wrong-topology": { // PostFilter tries to get the pod scheduleable by // deallocating the claim. pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{allocatedDelayedClaimWithWrongTopology}, + claims: []*resourceapi.ResourceClaim{allocatedClaimWithWrongTopology}, want: want{ filter: perNodeResult{ workerNode.Name: { @@ -1055,29 +986,11 @@ func TestPlugin(t *testing.T) { }, }, }, - "wrong-topology-immediate-allocation": { + "wrong-topology-structured": { // PostFilter tries to get the pod scheduleable by // deallocating the claim. pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{allocatedImmediateClaimWithWrongTopology}, - want: want{ - filter: perNodeResult{ - workerNode.Name: { - status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `resourceclaim not available on the node`), - }, - }, - postfilter: result{ - // Claims with immediate allocation don't. They would just get allocated again right - // away, without considering the needs of the pod. - status: framework.NewStatus(framework.Unschedulable, `still not schedulable`), - }, - }, - }, - "wrong-topology-delayed-allocation-structured": { - // PostFilter tries to get the pod scheduleable by - // deallocating the claim. - pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{structuredAllocatedDelayedClaimWithWrongTopology}, + claims: []*resourceapi.ResourceClaim{structuredAllocatedClaimWithWrongTopology}, want: want{ filter: perNodeResult{ workerNode.Name: { @@ -1097,25 +1010,6 @@ func TestPlugin(t *testing.T) { }, }, }, - "wrong-topology-immediate-allocation-structured": { - // PostFilter tries to get the pod scheduleable by - // deallocating the claim. - pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{structuredAllocatedImmediateClaimWithWrongTopology}, - want: want{ - filter: perNodeResult{ - workerNode.Name: { - status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `resourceclaim not available on the node`), - }, - }, - postfilter: result{ - // Claims with immediate allocation don't. The allocation is considered - // more important than the pod and pods need to wait for the node to - // become available again. - status: framework.NewStatus(framework.Unschedulable, `still not schedulable`), - }, - }, - }, "good-topology": { pod: podWithClaimName, claims: []*resourceapi.ResourceClaim{allocatedClaimWithGoodTopology}, @@ -1665,22 +1559,22 @@ func Test_isSchedulableAfterClaimChange(t *testing.T) { }, "queue-on-add": { pod: podWithClaimName, - newObj: pendingImmediateClaim, + newObj: pendingClaim, expectedHint: framework.Queue, }, "backoff-wrong-old-object": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, oldObj: "not-a-claim", - newObj: pendingImmediateClaim, + newObj: pendingClaim, expectedErr: true, }, "skip-adding-finalizer": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingImmediateClaim}, - oldObj: pendingImmediateClaim, + claims: []*resourceapi.ResourceClaim{pendingClaim}, + oldObj: pendingClaim, newObj: func() *resourceapi.ResourceClaim { - claim := pendingImmediateClaim.DeepCopy() + claim := pendingClaim.DeepCopy() claim.Finalizers = append(claim.Finalizers, "foo") return claim }(), @@ -1688,10 +1582,10 @@ func Test_isSchedulableAfterClaimChange(t *testing.T) { }, "queue-on-status-change": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingImmediateClaim}, - oldObj: pendingImmediateClaim, + claims: []*resourceapi.ResourceClaim{pendingClaim}, + oldObj: pendingClaim, newObj: func() *resourceapi.ResourceClaim { - claim := pendingImmediateClaim.DeepCopy() + claim := pendingClaim.DeepCopy() claim.Status.Allocation = &resourceapi.AllocationResult{} return claim }(), @@ -1699,7 +1593,7 @@ func Test_isSchedulableAfterClaimChange(t *testing.T) { }, "structured-claim-deallocate": { pod: podWithClaimName, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim, otherStructuredAllocatedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim, otherStructuredAllocatedClaim}, oldObj: otherStructuredAllocatedClaim, newObj: func() *resourceapi.ResourceClaim { claim := otherStructuredAllocatedClaim.DeepCopy() @@ -1804,7 +1698,7 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) { }, "skip-unrelated-object": { pod: podWithClaimTemplate, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, newObj: func() *resourceapi.PodSchedulingContext { scheduling := scheduling.DeepCopy() scheduling.Name += "-foo" @@ -1826,21 +1720,21 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) { }, "skip-missing-infos": { pod: podWithClaimTemplateInStatus, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, oldObj: scheduling, newObj: scheduling, expectedHint: framework.QueueSkip, }, "queue-new-infos": { pod: podWithClaimTemplateInStatus, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, oldObj: scheduling, newObj: schedulingInfo, expectedHint: framework.Queue, }, "queue-bad-selected-node": { pod: podWithClaimTemplateInStatus, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, oldObj: func() *resourceapi.PodSchedulingContext { scheduling := schedulingInfo.DeepCopy() scheduling.Spec.SelectedNode = workerNode.Name @@ -1856,7 +1750,7 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) { }, "skip-spec-changes": { pod: podWithClaimTemplateInStatus, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, oldObj: schedulingInfo, newObj: func() *resourceapi.PodSchedulingContext { scheduling := schedulingInfo.DeepCopy() @@ -1867,7 +1761,7 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) { }, "backoff-other-changes": { pod: podWithClaimTemplateInStatus, - claims: []*resourceapi.ResourceClaim{pendingDelayedClaim}, + claims: []*resourceapi.ResourceClaim{pendingClaim}, oldObj: schedulingInfo, newObj: func() *resourceapi.PodSchedulingContext { scheduling := schedulingInfo.DeepCopy() diff --git a/pkg/scheduler/testing/wrappers.go b/pkg/scheduler/testing/wrappers.go index 84fcf594213..03cdbd4f3f2 100644 --- a/pkg/scheduler/testing/wrappers.go +++ b/pkg/scheduler/testing/wrappers.go @@ -946,12 +946,6 @@ func (wrapper *ResourceClaimWrapper) OwnerReference(name, uid string, gvk schema return wrapper } -// AllocationMode sets the allocation mode of the inner object. -func (wrapper *ResourceClaimWrapper) AllocationMode(a resourceapi.AllocationMode) *ResourceClaimWrapper { - wrapper.ResourceClaim.Spec.AllocationMode = a - return wrapper -} - // ParametersRef sets a reference to a ResourceClaimParameters.resource.k8s.io. func (wrapper *ResourceClaimWrapper) ParametersRef(name string) *ResourceClaimWrapper { wrapper.ResourceClaim.Spec.ParametersRef = &resourceapi.ResourceClaimParametersReference{ diff --git a/staging/src/k8s.io/api/resource/v1alpha3/generated.pb.go b/staging/src/k8s.io/api/resource/v1alpha3/generated.pb.go index d3a9ec57d0e..a1534e0d30a 100644 --- a/staging/src/k8s.io/api/resource/v1alpha3/generated.pb.go +++ b/staging/src/k8s.io/api/resource/v1alpha3/generated.pb.go @@ -1331,148 +1331,146 @@ func init() { } var fileDescriptor_66649ee9bbcd89d2 = []byte{ - // 2242 bytes of a gzipped FileDescriptorProto + // 2219 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x1a, 0x4d, 0x6c, 0x1c, 0x57, - 0xd9, 0xb3, 0xbb, 0x89, 0xd7, 0x9f, 0xed, 0xb5, 0x33, 0xb6, 0xe3, 0x4d, 0xea, 0xee, 0x6e, 0x47, - 0x20, 0x2c, 0x70, 0x76, 0x1b, 0xa7, 0x4d, 0xa3, 0x52, 0x90, 0x32, 0x71, 0x13, 0x2c, 0x9a, 0xd4, - 0x7d, 0x4b, 0xdc, 0xa6, 0xfc, 0x75, 0xbc, 0xf3, 0x62, 0x0f, 0xd9, 0x9d, 0xd9, 0xcc, 0x7b, 0xeb, - 0x26, 0xe2, 0x12, 0x55, 0x20, 0xb8, 0x20, 0x15, 0x81, 0x10, 0x9c, 0x38, 0x21, 0xc4, 0x85, 0x0b, - 0x5c, 0x39, 0x55, 0xd0, 0x1c, 0x83, 0x40, 0xa2, 0xe2, 0xb0, 0x22, 0xcb, 0x91, 0x23, 0xb7, 0x9e, - 0xd0, 0xbc, 0xf7, 0xe6, 0xe7, 0xcd, 0xce, 0xac, 0x77, 0x96, 0xc6, 0x4a, 0x4e, 0xde, 0x79, 0xef, - 0xfb, 0x7b, 0xdf, 0xff, 0x7b, 0x9f, 0x61, 0xe3, 0xce, 0x25, 0x52, 0xb7, 0x9c, 0x86, 0xd1, 0xb5, - 0x1a, 0x2e, 0x26, 0x4e, 0xcf, 0x6d, 0xe1, 0xc6, 0xe1, 0x79, 0xa3, 0xdd, 0x3d, 0x30, 0x2e, 0x34, - 0xf6, 0xb1, 0x8d, 0x5d, 0x83, 0x62, 0xb3, 0xde, 0x75, 0x1d, 0xea, 0xa8, 0x6b, 0x1c, 0xba, 0x6e, - 0x74, 0xad, 0xba, 0x0f, 0x5d, 0xf7, 0xa1, 0xcf, 0x9e, 0xdb, 0xb7, 0xe8, 0x41, 0x6f, 0xaf, 0xde, - 0x72, 0x3a, 0x8d, 0x7d, 0x67, 0xdf, 0x69, 0x30, 0xa4, 0xbd, 0xde, 0x6d, 0xf6, 0xc5, 0x3e, 0xd8, - 0x2f, 0x4e, 0xec, 0xac, 0x16, 0x61, 0xdd, 0x72, 0x5c, 0x8f, 0x6d, 0x9c, 0xe1, 0xd9, 0x97, 0x42, - 0x98, 0x8e, 0xd1, 0x3a, 0xb0, 0x6c, 0xec, 0xde, 0x6f, 0x74, 0xef, 0xec, 0xcb, 0xf2, 0x66, 0xc1, - 0x22, 0x8d, 0x0e, 0xa6, 0x46, 0x12, 0xaf, 0x46, 0x1a, 0x96, 0xdb, 0xb3, 0xa9, 0xd5, 0x19, 0x66, - 0x73, 0xf1, 0x28, 0x04, 0xd2, 0x3a, 0xc0, 0x1d, 0x23, 0x8e, 0xa7, 0xfd, 0x32, 0x07, 0x8b, 0x97, - 0xdb, 0x6d, 0xa7, 0x65, 0x50, 0xcb, 0xb1, 0x11, 0x26, 0xbd, 0x36, 0x55, 0x1d, 0x58, 0xf0, 0xcf, - 0xf3, 0x35, 0xc3, 0x36, 0xdb, 0x98, 0x94, 0x95, 0x5a, 0x7e, 0x7d, 0x76, 0x73, 0xa3, 0x3e, 0x4a, - 0xe9, 0x75, 0x24, 0x21, 0xe9, 0xab, 0x0f, 0xfb, 0xd5, 0xa9, 0x41, 0xbf, 0xba, 0x20, 0xaf, 0x13, - 0x14, 0xa7, 0xae, 0xee, 0xc1, 0xa2, 0x71, 0x68, 0x58, 0x6d, 0x63, 0xaf, 0x8d, 0xdf, 0xb4, 0x6f, - 0x38, 0x26, 0x26, 0xe5, 0x5c, 0x4d, 0x59, 0x9f, 0xdd, 0xac, 0x45, 0x39, 0x7a, 0x96, 0xa9, 0x1f, - 0x9e, 0xaf, 0x7b, 0x00, 0x4d, 0xdc, 0xc6, 0x2d, 0xea, 0xb8, 0xfa, 0xf2, 0xa0, 0x5f, 0x5d, 0xbc, - 0x1c, 0xc3, 0x46, 0x43, 0xf4, 0xd4, 0x06, 0xcc, 0x90, 0x03, 0xc3, 0xc5, 0xde, 0x5a, 0x39, 0x5f, - 0x53, 0xd6, 0x8b, 0xfa, 0x29, 0x21, 0xe0, 0x4c, 0xd3, 0xdf, 0x40, 0x21, 0x8c, 0xf6, 0x53, 0x05, - 0x56, 0xe2, 0xaa, 0xb9, 0xee, 0x98, 0xb8, 0xad, 0xde, 0x83, 0x92, 0x6d, 0x74, 0xb0, 0xe9, 0x9f, - 0xcb, 0x53, 0x8f, 0x27, 0xec, 0x6b, 0xa3, 0xd5, 0x73, 0x43, 0xc2, 0x89, 0x93, 0xd6, 0xd5, 0x41, - 0xbf, 0x5a, 0x92, 0x61, 0x50, 0x8c, 0x8f, 0xf6, 0xfb, 0x1c, 0x9c, 0xde, 0x72, 0xad, 0x43, 0xec, - 0x0e, 0x19, 0xed, 0xc7, 0x0a, 0xac, 0x1e, 0x62, 0xdb, 0x74, 0x5c, 0x84, 0xef, 0xf6, 0x30, 0xa1, - 0x3b, 0x86, 0x6b, 0x74, 0x30, 0xc5, 0xae, 0x2f, 0xde, 0xb9, 0x88, 0x78, 0x81, 0x93, 0xd4, 0xbb, - 0x77, 0xf6, 0xeb, 0xc2, 0x49, 0xea, 0xc8, 0x78, 0xff, 0xf5, 0x7b, 0x14, 0xdb, 0xc4, 0x72, 0x6c, - 0xbd, 0x2a, 0xb4, 0xb3, 0xba, 0x9b, 0x4c, 0x15, 0xa5, 0xb1, 0xf3, 0x44, 0x59, 0x31, 0x92, 0x34, - 0x27, 0x8c, 0x7a, 0x61, 0xb4, 0x9e, 0x12, 0x95, 0xae, 0x3f, 0x2f, 0xc4, 0x49, 0xb6, 0x09, 0x4a, - 0x66, 0xa8, 0xfd, 0x22, 0x07, 0x25, 0xae, 0x30, 0x21, 0x26, 0x51, 0x37, 0x01, 0x4c, 0xb6, 0xe2, - 0xe9, 0x9a, 0xa9, 0x66, 0x46, 0x57, 0x05, 0x71, 0xd8, 0x0a, 0x76, 0x50, 0x04, 0x4a, 0x25, 0xb0, - 0xc8, 0x0f, 0x1b, 0x51, 0x6a, 0x6e, 0x12, 0xa5, 0x96, 0x05, 0xa3, 0xc5, 0xdd, 0x18, 0x39, 0x34, - 0xc4, 0x40, 0xfd, 0x26, 0x14, 0x5d, 0x21, 0x74, 0x39, 0xcf, 0xe2, 0xef, 0xdc, 0x78, 0xf1, 0x27, - 0x8e, 0xaa, 0x2f, 0x0a, 0x66, 0x45, 0xff, 0xec, 0x28, 0x20, 0xa8, 0xe9, 0x50, 0x19, 0xed, 0x8f, - 0x6a, 0x0d, 0x0a, 0x76, 0xa8, 0xa1, 0x39, 0x41, 0xab, 0xc0, 0x74, 0xc3, 0x76, 0xb4, 0xbf, 0x28, - 0xb0, 0x1a, 0x23, 0x42, 0xa9, 0x6b, 0xed, 0xf5, 0x28, 0x3e, 0x1a, 0xdb, 0xf3, 0x92, 0x92, 0xe1, - 0xc3, 0xef, 0x1a, 0xed, 0x1e, 0x16, 0x2a, 0x7d, 0x35, 0x53, 0x18, 0x49, 0x14, 0xf4, 0xcf, 0x09, - 0x46, 0x6b, 0xa3, 0xa0, 0x50, 0x8c, 0xaf, 0xf6, 0x9f, 0x3c, 0x8c, 0x44, 0x50, 0xbf, 0x0d, 0xc5, - 0xbb, 0x3d, 0xc3, 0xa6, 0x16, 0xbd, 0x5f, 0x3e, 0xc9, 0x84, 0xac, 0xa7, 0xda, 0x5d, 0x92, 0xfa, - 0x2d, 0x81, 0xa5, 0x9f, 0x1a, 0xf4, 0xab, 0xf3, 0xfe, 0x17, 0x97, 0x22, 0x20, 0xa9, 0xbe, 0x00, - 0x85, 0x3d, 0xc7, 0xe1, 0xe1, 0x51, 0xd4, 0xe7, 0xbd, 0x94, 0xa4, 0x3b, 0x4e, 0x9b, 0x83, 0xb1, - 0x2d, 0xb5, 0x02, 0x79, 0xcb, 0xa6, 0xe5, 0xe9, 0x9a, 0xb2, 0x9e, 0xd7, 0xe7, 0x3c, 0xa3, 0x6e, - 0xdb, 0x94, 0x03, 0x78, 0x1b, 0x6a, 0x0b, 0x8a, 0x96, 0x4d, 0x9b, 0x6d, 0xab, 0x85, 0xcb, 0x45, - 0x26, 0xe1, 0x4b, 0x59, 0xd4, 0xb8, 0x2d, 0x70, 0xb9, 0x9c, 0xfe, 0x97, 0x90, 0xd3, 0x27, 0xac, - 0x7e, 0x01, 0x4e, 0x12, 0xea, 0x5a, 0xf6, 0x7e, 0xf9, 0x04, 0x33, 0xeb, 0xc2, 0xa0, 0x5f, 0x9d, - 0x6d, 0xb2, 0x15, 0x0e, 0x2a, 0xb6, 0x55, 0x07, 0x66, 0xf9, 0x2f, 0x2e, 0xd0, 0x0c, 0x13, 0xe8, - 0x95, 0x2c, 0x02, 0x35, 0x43, 0x74, 0x9e, 0xe2, 0x23, 0x0b, 0x9c, 0x57, 0x94, 0x83, 0xfa, 0x45, - 0x98, 0x3e, 0xc4, 0xae, 0x17, 0x62, 0x65, 0x60, 0xa2, 0x2d, 0x0e, 0xfa, 0xd5, 0xb9, 0x5d, 0xbe, - 0xc4, 0xe1, 0x7d, 0x00, 0x6d, 0x0b, 0x96, 0x65, 0x5e, 0x57, 0xad, 0x36, 0xc5, 0xae, 0xba, 0x01, - 0x45, 0x22, 0xaa, 0x8a, 0x70, 0xdb, 0x20, 0x80, 0xfc, 0x6a, 0x83, 0x02, 0x08, 0xed, 0x37, 0x0a, - 0x9c, 0x8e, 0xeb, 0x90, 0x50, 0xc3, 0x6e, 0x8d, 0xe3, 0xfb, 0x16, 0x40, 0xe0, 0x82, 0x5e, 0x26, - 0xf1, 0x82, 0xfb, 0xe5, 0x89, 0xdc, 0x3e, 0x4c, 0x5d, 0xc1, 0x12, 0x41, 0x11, 0xe2, 0xda, 0xc5, - 0x61, 0x31, 0x85, 0x35, 0xd7, 0xa0, 0x60, 0xd9, 0x94, 0xd7, 0xf6, 0xbc, 0x5e, 0xf4, 0x44, 0xdc, - 0xb6, 0x29, 0x41, 0x6c, 0x55, 0x7b, 0x1d, 0x56, 0x62, 0xc5, 0x88, 0xa7, 0x8e, 0x8c, 0x6a, 0x7a, - 0x30, 0x94, 0x23, 0x82, 0x1f, 0x2a, 0x86, 0x19, 0x4b, 0xe8, 0xcc, 0xef, 0x30, 0x32, 0x3a, 0x2d, - 0x47, 0x0e, 0x0b, 0xb9, 0xbf, 0x42, 0x50, 0x48, 0x59, 0xd3, 0xe1, 0x4c, 0xaa, 0x6f, 0xa9, 0x9f, - 0x87, 0x69, 0xee, 0x47, 0x5c, 0x82, 0x19, 0x7d, 0x76, 0xd0, 0xaf, 0x4e, 0x73, 0x08, 0x82, 0xfc, - 0x3d, 0xed, 0x8f, 0x39, 0x58, 0xde, 0x71, 0xcc, 0x66, 0xeb, 0x00, 0x9b, 0xbd, 0xb6, 0x65, 0xef, - 0x5f, 0x71, 0x6c, 0x8a, 0xef, 0x51, 0xf5, 0x3d, 0x28, 0x7a, 0x4d, 0x9c, 0x69, 0x50, 0x43, 0x94, - 0xd9, 0x17, 0x47, 0x65, 0x06, 0x52, 0xf7, 0xa0, 0xbd, 0x26, 0xe6, 0xcd, 0xbd, 0xef, 0xe1, 0x16, - 0xbd, 0x8e, 0xa9, 0x11, 0x9a, 0x30, 0x5c, 0x43, 0x01, 0x55, 0xf5, 0x1d, 0x28, 0x90, 0x2e, 0x6e, - 0x89, 0xe4, 0x78, 0x71, 0xb4, 0x82, 0x92, 0x64, 0x6c, 0x76, 0x71, 0x2b, 0xf4, 0x42, 0xef, 0x0b, - 0x31, 0x8a, 0xea, 0x7b, 0x5e, 0x38, 0x1b, 0xb4, 0x47, 0x58, 0x3f, 0x34, 0xbb, 0x79, 0x69, 0x02, - 0xda, 0x0c, 0x5f, 0x2f, 0x09, 0xea, 0x27, 0xf9, 0x37, 0x12, 0x74, 0xb5, 0xbf, 0x2a, 0x50, 0x4e, - 0x42, 0x7b, 0xc3, 0x22, 0x54, 0xfd, 0xd6, 0x90, 0xea, 0xea, 0xe3, 0xa9, 0xce, 0xc3, 0x66, 0x8a, - 0x0b, 0x1c, 0xcf, 0x5f, 0x89, 0xa8, 0xed, 0x6d, 0x38, 0x61, 0x51, 0xdc, 0xf1, 0xa3, 0x6b, 0x33, - 0xfb, 0xd9, 0xf4, 0x79, 0x41, 0xfe, 0xc4, 0xb6, 0x47, 0x08, 0x71, 0x7a, 0xda, 0x87, 0x29, 0x67, - 0xf2, 0x14, 0xab, 0x5e, 0x82, 0x39, 0xee, 0xfa, 0xd8, 0xf4, 0xda, 0x4e, 0x11, 0x20, 0xcb, 0x82, - 0xd0, 0x5c, 0x33, 0xb2, 0x87, 0x24, 0x48, 0xf5, 0x55, 0x28, 0x75, 0x1d, 0x8a, 0x6d, 0x6a, 0x19, - 0x6d, 0xbf, 0x03, 0xf6, 0xfc, 0x91, 0xb5, 0x85, 0x3b, 0xd2, 0x0e, 0x8a, 0x41, 0x6a, 0xbf, 0x52, - 0xe0, 0x6c, 0xba, 0x75, 0xd4, 0xef, 0x43, 0xc9, 0x3f, 0xf1, 0x95, 0xb6, 0x61, 0x75, 0xfc, 0x60, - 0xfb, 0xf2, 0x78, 0xed, 0x04, 0xc3, 0x09, 0x69, 0x0b, 0x93, 0x9f, 0x16, 0x67, 0x2a, 0x49, 0x60, - 0x04, 0xc5, 0x58, 0x69, 0xbf, 0xce, 0xc1, 0xbc, 0x04, 0x72, 0x0c, 0x21, 0xf3, 0x96, 0x14, 0x32, - 0x8d, 0x2c, 0xc7, 0x4c, 0x8b, 0x95, 0x5b, 0xb1, 0x58, 0x39, 0x9f, 0x85, 0xe8, 0xe8, 0x20, 0x19, - 0x28, 0x50, 0x91, 0xe0, 0xaf, 0x38, 0x36, 0xe9, 0x75, 0xbc, 0x96, 0xf5, 0x36, 0x76, 0xb1, 0x57, - 0x51, 0x36, 0xa0, 0x68, 0x74, 0xad, 0x6b, 0xae, 0xd3, 0xeb, 0xc6, 0x73, 0xee, 0xe5, 0x9d, 0x6d, - 0xb6, 0x8e, 0x02, 0x08, 0x0f, 0xda, 0x97, 0x88, 0x49, 0x3b, 0x13, 0xed, 0x04, 0x45, 0x8b, 0x18, - 0x40, 0x04, 0xd5, 0xaa, 0x90, 0x5a, 0xad, 0x74, 0xc8, 0xf7, 0x2c, 0x53, 0xd4, 0xfc, 0x17, 0x05, - 0x40, 0xfe, 0xe6, 0xf6, 0xd6, 0xa7, 0xfd, 0xea, 0x0b, 0x69, 0x17, 0x4f, 0x7a, 0xbf, 0x8b, 0x49, - 0xfd, 0xe6, 0xf6, 0x16, 0xf2, 0x90, 0xb5, 0x8f, 0x14, 0x38, 0x25, 0x1d, 0xf2, 0x18, 0x52, 0xc0, - 0x8e, 0x9c, 0x02, 0xbe, 0x94, 0xc1, 0x64, 0x29, 0xb1, 0xff, 0xb3, 0x3c, 0xac, 0x4a, 0x70, 0x91, - 0x76, 0xfd, 0xc9, 0xbb, 0xf5, 0xfb, 0x30, 0x1f, 0xdc, 0xdf, 0xaf, 0xba, 0x4e, 0x47, 0xf8, 0xf7, - 0x57, 0x33, 0x9c, 0x2b, 0x72, 0xe1, 0xf0, 0x9d, 0x8b, 0xb7, 0x7c, 0xd7, 0xa2, 0x84, 0x91, 0xcc, - 0x27, 0xf3, 0xdd, 0x59, 0x6d, 0x43, 0xc9, 0x94, 0x6e, 0x5d, 0xe5, 0xc2, 0x38, 0x0f, 0x08, 0xf2, - 0x4d, 0x2d, 0x4c, 0x31, 0xf2, 0x3a, 0x8a, 0xd1, 0xd6, 0xfe, 0xa1, 0xc0, 0x73, 0x29, 0xa7, 0x3c, - 0x06, 0x2f, 0x7b, 0x57, 0xf6, 0xb2, 0x97, 0x27, 0xb2, 0x46, 0x8a, 0xbf, 0xfd, 0x5c, 0x81, 0xda, - 0x51, 0xf6, 0xcb, 0x98, 0x1c, 0x6a, 0x50, 0xb8, 0x63, 0xd9, 0x26, 0xf3, 0x9d, 0x48, 0xb8, 0x7f, - 0xdd, 0xb2, 0x4d, 0xc4, 0x76, 0x82, 0x84, 0x90, 0x4f, 0xbd, 0xf8, 0x3d, 0x50, 0xe0, 0xf9, 0x91, - 0xd5, 0x61, 0x8c, 0x16, 0xf8, 0x2b, 0xb0, 0xd0, 0xb3, 0x49, 0xcf, 0xa2, 0x9e, 0xc3, 0x44, 0x0b, - 0xde, 0xd2, 0xa0, 0x5f, 0x5d, 0xb8, 0x29, 0x6f, 0xa1, 0x38, 0xac, 0xf6, 0xdb, 0x5c, 0x2c, 0x9f, - 0xb0, 0xf2, 0x7b, 0x0d, 0x4e, 0x45, 0xca, 0x0f, 0x21, 0x91, 0x2b, 0xfe, 0x19, 0x21, 0x43, 0x14, - 0x8b, 0x03, 0xa0, 0x61, 0x1c, 0x2f, 0xd4, 0xba, 0x51, 0x55, 0x7f, 0x96, 0xa1, 0x26, 0x6d, 0x20, - 0x99, 0x8f, 0xba, 0x03, 0xa5, 0xf0, 0x25, 0xe3, 0xba, 0xd7, 0x42, 0x70, 0x33, 0xac, 0xfb, 0xb1, - 0x70, 0x59, 0xda, 0xfd, 0x74, 0x68, 0x05, 0xc5, 0xf0, 0xb5, 0xff, 0xe6, 0x60, 0x29, 0xa1, 0x1c, - 0x4d, 0xf4, 0x0e, 0xf2, 0x1d, 0x80, 0x90, 0xba, 0xd0, 0x49, 0x3d, 0xdb, 0x6b, 0x8e, 0x5e, 0x62, - 0x97, 0x95, 0x70, 0x35, 0x42, 0x51, 0x25, 0x30, 0xeb, 0x62, 0x82, 0xdd, 0x43, 0x6c, 0x5e, 0x75, - 0x5c, 0xf1, 0xea, 0xf1, 0x5a, 0x06, 0xa5, 0x0f, 0x95, 0x4e, 0x7d, 0x49, 0x1c, 0x69, 0x16, 0x85, - 0x84, 0x51, 0x94, 0x8b, 0xda, 0x84, 0x15, 0x13, 0x47, 0x9f, 0x8f, 0x58, 0x5a, 0xc1, 0x26, 0xab, - 0x88, 0xc5, 0xf0, 0xe1, 0x69, 0x2b, 0x09, 0x08, 0x25, 0xe3, 0x6a, 0x7f, 0x57, 0x60, 0x45, 0x92, - 0xec, 0x1b, 0xb8, 0xd3, 0x6d, 0x1b, 0x14, 0x1f, 0x43, 0x9d, 0xb8, 0x25, 0xb5, 0x3f, 0xaf, 0x64, - 0x50, 0x9f, 0x2f, 0x64, 0x5a, 0x1b, 0xa4, 0xfd, 0x4d, 0x81, 0x33, 0x89, 0x18, 0xc7, 0x90, 0x68, - 0xdf, 0x91, 0x13, 0xed, 0x85, 0x09, 0xce, 0x95, 0x92, 0x66, 0x1f, 0xa5, 0x9d, 0xaa, 0xc9, 0xaf, - 0x49, 0xcf, 0x5e, 0xbf, 0xaa, 0x7d, 0x9c, 0x97, 0xda, 0x6e, 0x72, 0x1c, 0xfd, 0x89, 0x9c, 0x51, - 0x72, 0x63, 0x65, 0x94, 0xa1, 0x44, 0x9b, 0xcf, 0x98, 0x68, 0x09, 0x99, 0x2c, 0xd1, 0xde, 0x82, - 0x79, 0xb9, 0xfa, 0x14, 0xc6, 0x1c, 0x38, 0x30, 0xd2, 0x4d, 0xa9, 0x3a, 0xc9, 0x94, 0xd4, 0x37, - 0x60, 0x99, 0x50, 0xb7, 0xd7, 0xa2, 0x3d, 0x17, 0x9b, 0x91, 0x17, 0xe3, 0x13, 0x2c, 0x9f, 0x94, - 0x07, 0xfd, 0xea, 0x72, 0x33, 0x61, 0x1f, 0x25, 0x62, 0xc5, 0x3b, 0x67, 0x42, 0x9e, 0xe6, 0xce, - 0x99, 0xa4, 0x75, 0x32, 0x1f, 0xc9, 0x9d, 0x73, 0xd4, 0x6a, 0xcf, 0x42, 0xe7, 0x3c, 0xc2, 0xcb, - 0x46, 0x76, 0xce, 0x34, 0x61, 0x70, 0xc0, 0xab, 0xda, 0x11, 0x65, 0x33, 0x3e, 0x1f, 0xc8, 0x34, - 0x39, 0x78, 0x1b, 0xa6, 0x6f, 0xb3, 0x37, 0xcd, 0x31, 0xfb, 0x6e, 0xff, 0xa0, 0xfc, 0x21, 0x54, - 0x5f, 0x10, 0xac, 0xa6, 0xf9, 0x37, 0x41, 0x3e, 0xb5, 0x78, 0xa7, 0x1d, 0xd5, 0xca, 0xd3, 0xdc, - 0x69, 0x47, 0xe5, 0x4c, 0xf1, 0xcf, 0x3f, 0xcb, 0x9d, 0x76, 0xa2, 0xbd, 0x8f, 0xbf, 0xd3, 0xf6, - 0x6e, 0x5e, 0xde, 0x5f, 0xd2, 0x35, 0x5a, 0xfe, 0x0d, 0x3d, 0xb8, 0x79, 0xdd, 0xf0, 0x37, 0x50, - 0x08, 0xa3, 0x7d, 0xac, 0x40, 0x49, 0x36, 0xe7, 0x44, 0x8d, 0xde, 0x03, 0x05, 0x96, 0x5c, 0x89, - 0x4c, 0x74, 0x80, 0x77, 0x3e, 0x8b, 0x3b, 0xf1, 0xf1, 0xdd, 0x73, 0x82, 0xe1, 0x52, 0xc2, 0x26, - 0x4a, 0x62, 0xa5, 0xfd, 0x50, 0x81, 0x24, 0x60, 0xd5, 0x4e, 0x99, 0xbe, 0x6e, 0x66, 0x79, 0x3a, - 0x16, 0x9e, 0x3e, 0xce, 0xcc, 0xf5, 0x9f, 0x11, 0x8d, 0xf2, 0x81, 0xf5, 0x44, 0x1a, 0xad, 0x41, - 0x81, 0x85, 0x45, 0xcc, 0x1b, 0xb6, 0x0c, 0x6a, 0x20, 0xb6, 0xa3, 0xba, 0x50, 0x0a, 0x0b, 0x80, - 0xb7, 0xce, 0x0a, 0xc6, 0x91, 0x4f, 0xbe, 0x61, 0x29, 0x89, 0xcd, 0xdf, 0xd9, 0xe1, 0x9a, 0x12, - 0x45, 0x14, 0xe3, 0xa0, 0x7d, 0xa0, 0x84, 0x6d, 0x02, 0x57, 0xef, 0xdd, 0x14, 0xf5, 0x66, 0x1a, - 0x4f, 0x04, 0x3f, 0xc6, 0xd2, 0xf0, 0x4f, 0x72, 0xb0, 0x10, 0x9b, 0x5d, 0x26, 0x4e, 0x5c, 0x95, - 0x27, 0x3d, 0x71, 0xfd, 0x81, 0x02, 0xcb, 0xae, 0x2c, 0x48, 0xd4, 0xed, 0x37, 0x33, 0x8d, 0x5f, - 0xb9, 0xdf, 0xaf, 0x09, 0xf6, 0xcb, 0x49, 0xbb, 0x28, 0x91, 0x9b, 0xf6, 0x23, 0x05, 0x12, 0xc1, - 0x55, 0x27, 0xc5, 0x36, 0x17, 0xb2, 0xd9, 0x86, 0x4f, 0x87, 0xc7, 0xb1, 0xcc, 0x9f, 0x22, 0x8f, - 0xb7, 0x7c, 0x5e, 0xf2, 0xe4, 0x6b, 0xf5, 0x06, 0x14, 0x6d, 0xc7, 0xc4, 0x91, 0x1e, 0x32, 0x48, - 0xb2, 0x37, 0xc4, 0x3a, 0x0a, 0x20, 0x62, 0xa1, 0x98, 0x1f, 0x2b, 0x14, 0x0f, 0x60, 0xde, 0x8d, - 0xfa, 0xbc, 0x68, 0xfd, 0xc6, 0xec, 0x72, 0xb8, 0x5d, 0x57, 0x04, 0x0f, 0x39, 0x7a, 0x90, 0x4c, - 0x58, 0xea, 0xdd, 0x98, 0xfe, 0x9e, 0xda, 0xde, 0x8d, 0x4f, 0x5a, 0x93, 0x6b, 0xe3, 0x1f, 0xf2, - 0x50, 0x4e, 0xcb, 0x32, 0xea, 0x07, 0x0a, 0xac, 0xf0, 0x40, 0x8a, 0x95, 0xcd, 0xc9, 0xc2, 0x35, - 0xb8, 0x6d, 0xef, 0x26, 0xd1, 0x44, 0xc9, 0xac, 0x64, 0x21, 0xa2, 0x4f, 0x2f, 0x93, 0xfd, 0x97, - 0xc6, 0xb0, 0x10, 0xd2, 0x73, 0x4e, 0x32, 0x2b, 0xc9, 0x71, 0x0b, 0x47, 0x3a, 0xee, 0x77, 0x61, - 0xda, 0x65, 0x0f, 0x22, 0xde, 0xbd, 0x60, 0x8c, 0xd1, 0x67, 0xf2, 0xbf, 0xfd, 0x84, 0xbd, 0x1a, - 0xff, 0x26, 0xc8, 0xa7, 0xaa, 0xfd, 0x4e, 0x81, 0xa1, 0x9c, 0x37, 0x51, 0xe5, 0x32, 0x00, 0xba, - 0xff, 0xa7, 0x42, 0x03, 0x16, 0x11, 0x2d, 0x46, 0x88, 0xea, 0xfa, 0xc3, 0xc7, 0x95, 0xa9, 0x47, - 0x8f, 0x2b, 0x53, 0x9f, 0x3c, 0xae, 0x4c, 0x3d, 0x18, 0x54, 0x94, 0x87, 0x83, 0x8a, 0xf2, 0x68, - 0x50, 0x51, 0x3e, 0x19, 0x54, 0x94, 0x7f, 0x0d, 0x2a, 0xca, 0x87, 0xff, 0xae, 0x4c, 0xbd, 0xbb, - 0x36, 0xea, 0x1f, 0x04, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x46, 0x21, 0x2d, 0x8a, 0x3f, 0x28, - 0x00, 0x00, + 0xd9, 0xb3, 0xbb, 0x89, 0xd7, 0x9f, 0xed, 0xb5, 0x33, 0xb6, 0xe3, 0x4d, 0xea, 0xee, 0xba, 0x23, + 0x10, 0x11, 0x38, 0xbb, 0x8d, 0xd3, 0xa6, 0x51, 0x29, 0x48, 0x99, 0xb8, 0x09, 0x16, 0x6d, 0xea, + 0xbe, 0x25, 0x6e, 0x53, 0xfe, 0x3a, 0xde, 0x79, 0xb1, 0x87, 0xec, 0xce, 0x6c, 0xe6, 0xbd, 0x71, + 0x13, 0x71, 0x89, 0x2a, 0x10, 0x5c, 0x90, 0x8a, 0x40, 0x08, 0x4e, 0x9c, 0x38, 0x70, 0xe1, 0x02, + 0x57, 0x4e, 0x15, 0x34, 0xc7, 0x20, 0x40, 0x54, 0x1c, 0x56, 0x64, 0x39, 0x72, 0xe4, 0xc6, 0x09, + 0xcd, 0x7b, 0x6f, 0x7e, 0xde, 0xec, 0xcc, 0x7a, 0x67, 0x21, 0x56, 0x72, 0xf2, 0xce, 0x7b, 0xdf, + 0xdf, 0xfb, 0xfe, 0xdf, 0xf7, 0x0c, 0x1b, 0x77, 0x2e, 0x93, 0x86, 0xe5, 0x34, 0x8d, 0x9e, 0xd5, + 0x74, 0x31, 0x71, 0x3c, 0xb7, 0x8d, 0x9b, 0x87, 0x17, 0x8c, 0x4e, 0xef, 0xc0, 0xb8, 0xd8, 0xdc, + 0xc7, 0x36, 0x76, 0x0d, 0x8a, 0xcd, 0x46, 0xcf, 0x75, 0xa8, 0xa3, 0xae, 0x71, 0xe8, 0x86, 0xd1, + 0xb3, 0x1a, 0x01, 0x74, 0x23, 0x80, 0x3e, 0x7b, 0x7e, 0xdf, 0xa2, 0x07, 0xde, 0x5e, 0xa3, 0xed, + 0x74, 0x9b, 0xfb, 0xce, 0xbe, 0xd3, 0x64, 0x48, 0x7b, 0xde, 0x6d, 0xf6, 0xc5, 0x3e, 0xd8, 0x2f, + 0x4e, 0xec, 0xac, 0x16, 0x63, 0xdd, 0x76, 0x5c, 0x9f, 0x6d, 0x92, 0xe1, 0xd9, 0x97, 0x22, 0x98, + 0xae, 0xd1, 0x3e, 0xb0, 0x6c, 0xec, 0xde, 0x6f, 0xf6, 0xee, 0xec, 0xcb, 0xf2, 0xe6, 0xc1, 0x22, + 0xcd, 0x2e, 0xa6, 0x46, 0x1a, 0xaf, 0x66, 0x16, 0x96, 0xeb, 0xd9, 0xd4, 0xea, 0x0e, 0xb3, 0xb9, + 0x74, 0x14, 0x02, 0x69, 0x1f, 0xe0, 0xae, 0x91, 0xc4, 0xd3, 0x7e, 0x5e, 0x80, 0xc5, 0x2b, 0x9d, + 0x8e, 0xd3, 0x36, 0xa8, 0xe5, 0xd8, 0x08, 0x13, 0xaf, 0x43, 0x55, 0x07, 0x16, 0x82, 0xf3, 0x7c, + 0xc5, 0xb0, 0xcd, 0x0e, 0x26, 0x55, 0x65, 0xbd, 0x78, 0x6e, 0x76, 0x73, 0xa3, 0x31, 0x4a, 0xe9, + 0x0d, 0x24, 0x21, 0xe9, 0xab, 0x0f, 0xfb, 0xf5, 0xa9, 0x41, 0xbf, 0xbe, 0x20, 0xaf, 0x13, 0x94, + 0xa4, 0xae, 0xee, 0xc1, 0xa2, 0x71, 0x68, 0x58, 0x1d, 0x63, 0xaf, 0x83, 0xdf, 0xb2, 0x6f, 0x38, + 0x26, 0x26, 0xd5, 0xc2, 0xba, 0x72, 0x6e, 0x76, 0x73, 0x3d, 0xce, 0xd1, 0xb7, 0x4c, 0xe3, 0xf0, + 0x42, 0xc3, 0x07, 0x68, 0xe1, 0x0e, 0x6e, 0x53, 0xc7, 0xd5, 0x97, 0x07, 0xfd, 0xfa, 0xe2, 0x95, + 0x04, 0x36, 0x1a, 0xa2, 0xa7, 0x36, 0x61, 0x86, 0x1c, 0x18, 0x2e, 0xf6, 0xd7, 0xaa, 0xc5, 0x75, + 0xe5, 0x5c, 0x59, 0x3f, 0x25, 0x04, 0x9c, 0x69, 0x05, 0x1b, 0x28, 0x82, 0xd1, 0x7e, 0xac, 0xc0, + 0x4a, 0x52, 0x35, 0x6f, 0x3a, 0x26, 0xee, 0xa8, 0xf7, 0xa0, 0x62, 0x1b, 0x5d, 0x6c, 0x06, 0xe7, + 0xf2, 0xd5, 0xe3, 0x0b, 0xfb, 0xda, 0x68, 0xf5, 0xdc, 0x90, 0x70, 0x92, 0xa4, 0x75, 0x75, 0xd0, + 0xaf, 0x57, 0x64, 0x18, 0x94, 0xe0, 0xa3, 0xfd, 0xa6, 0x00, 0xa7, 0xb7, 0x5c, 0xeb, 0x10, 0xbb, + 0x43, 0x46, 0xfb, 0xa1, 0x02, 0xab, 0x87, 0xd8, 0x36, 0x1d, 0x17, 0xe1, 0xbb, 0x1e, 0x26, 0x74, + 0xc7, 0x70, 0x8d, 0x2e, 0xa6, 0xd8, 0x0d, 0xc4, 0x3b, 0x1f, 0x13, 0x2f, 0x74, 0x92, 0x46, 0xef, + 0xce, 0x7e, 0x43, 0x38, 0x49, 0x03, 0x19, 0x1f, 0xbc, 0x7e, 0x8f, 0x62, 0x9b, 0x58, 0x8e, 0xad, + 0xd7, 0x85, 0x76, 0x56, 0x77, 0xd3, 0xa9, 0xa2, 0x2c, 0x76, 0xbe, 0x28, 0x2b, 0x46, 0x9a, 0xe6, + 0x84, 0x51, 0x2f, 0x8e, 0xd6, 0x53, 0xaa, 0xd2, 0xf5, 0xe7, 0x85, 0x38, 0xe9, 0x36, 0x41, 0xe9, + 0x0c, 0xb5, 0x9f, 0x15, 0xa0, 0xc2, 0x15, 0x26, 0xc4, 0x24, 0xea, 0x26, 0x80, 0xc9, 0x56, 0x7c, + 0x5d, 0x33, 0xd5, 0xcc, 0xe8, 0xaa, 0x20, 0x0e, 0x5b, 0xe1, 0x0e, 0x8a, 0x41, 0xa9, 0x04, 0x16, + 0xf9, 0x61, 0x63, 0x4a, 0x2d, 0x4c, 0xa2, 0xd4, 0xaa, 0x60, 0xb4, 0xb8, 0x9b, 0x20, 0x87, 0x86, + 0x18, 0xa8, 0x5f, 0x87, 0xb2, 0x2b, 0x84, 0xae, 0x16, 0x59, 0xfc, 0x9d, 0x1f, 0x2f, 0xfe, 0xc4, + 0x51, 0xf5, 0x45, 0xc1, 0xac, 0x1c, 0x9c, 0x1d, 0x85, 0x04, 0x35, 0x1d, 0x6a, 0xa3, 0xfd, 0x51, + 0x5d, 0x87, 0x92, 0x1d, 0x69, 0x68, 0x4e, 0xd0, 0x2a, 0x31, 0xdd, 0xb0, 0x1d, 0xed, 0x8f, 0x0a, + 0xac, 0x26, 0x88, 0x50, 0xea, 0x5a, 0x7b, 0x1e, 0xc5, 0x47, 0x63, 0xfb, 0x5e, 0x52, 0x31, 0x02, + 0xf8, 0x5d, 0xa3, 0xe3, 0x61, 0xa1, 0xd2, 0x57, 0x73, 0x85, 0x91, 0x44, 0x41, 0xff, 0x8c, 0x60, + 0xb4, 0x36, 0x0a, 0x0a, 0x25, 0xf8, 0x6a, 0xff, 0x2a, 0xc2, 0x48, 0x04, 0xf5, 0x9b, 0x50, 0xbe, + 0xeb, 0x19, 0x36, 0xb5, 0xe8, 0xfd, 0xea, 0x49, 0x26, 0x64, 0x23, 0xd3, 0xee, 0x92, 0xd4, 0x6f, + 0x0b, 0x2c, 0xfd, 0xd4, 0xa0, 0x5f, 0x9f, 0x0f, 0xbe, 0xb8, 0x14, 0x21, 0x49, 0xf5, 0x05, 0x28, + 0xed, 0x39, 0x0e, 0x0f, 0x8f, 0xb2, 0x3e, 0xef, 0xa7, 0x24, 0xdd, 0x71, 0x3a, 0x1c, 0x8c, 0x6d, + 0xa9, 0x35, 0x28, 0x5a, 0x36, 0xad, 0x4e, 0xaf, 0x2b, 0xe7, 0x8a, 0xfa, 0x9c, 0x6f, 0xd4, 0x6d, + 0x9b, 0x72, 0x00, 0x7f, 0x43, 0x6d, 0x43, 0xd9, 0xb2, 0x69, 0xab, 0x63, 0xb5, 0x71, 0xb5, 0xcc, + 0x24, 0x7c, 0x29, 0x8f, 0x1a, 0xb7, 0x05, 0x2e, 0x97, 0x33, 0xf8, 0x12, 0x72, 0x06, 0x84, 0xd5, + 0xcf, 0xc1, 0x49, 0x42, 0x5d, 0xcb, 0xde, 0xaf, 0x9e, 0x60, 0x66, 0x5d, 0x18, 0xf4, 0xeb, 0xb3, + 0x2d, 0xb6, 0xc2, 0x41, 0xc5, 0xb6, 0xea, 0xc0, 0x2c, 0xff, 0xc5, 0x05, 0x9a, 0x61, 0x02, 0xbd, + 0x92, 0x47, 0xa0, 0x56, 0x84, 0xce, 0x53, 0x7c, 0x6c, 0x81, 0xf3, 0x8a, 0x73, 0x50, 0x3f, 0x0f, + 0xd3, 0x87, 0xd8, 0xf5, 0x43, 0xac, 0x0a, 0x4c, 0xb4, 0xc5, 0x41, 0xbf, 0x3e, 0xb7, 0xcb, 0x97, + 0x38, 0x7c, 0x00, 0xa0, 0x6d, 0xc1, 0xb2, 0xcc, 0xeb, 0x9a, 0xd5, 0xa1, 0xd8, 0x55, 0x37, 0xa0, + 0x4c, 0x44, 0x55, 0x11, 0x6e, 0x1b, 0x06, 0x50, 0x50, 0x6d, 0x50, 0x08, 0xa1, 0xfd, 0x4a, 0x81, + 0xd3, 0x49, 0x1d, 0x12, 0x6a, 0xd8, 0xed, 0x71, 0x7c, 0xdf, 0x02, 0x08, 0x5d, 0xd0, 0xcf, 0x24, + 0x7e, 0x70, 0xbf, 0x3c, 0x91, 0xdb, 0x47, 0xa9, 0x2b, 0x5c, 0x22, 0x28, 0x46, 0x5c, 0xbb, 0x34, + 0x2c, 0xa6, 0xb0, 0xe6, 0x1a, 0x94, 0x2c, 0x9b, 0xf2, 0xda, 0x5e, 0xd4, 0xcb, 0xbe, 0x88, 0xdb, + 0x36, 0x25, 0x88, 0xad, 0x6a, 0xaf, 0xc3, 0x4a, 0xa2, 0x18, 0xf1, 0xd4, 0x91, 0x53, 0x4d, 0x0f, + 0x86, 0x72, 0x44, 0xf8, 0x43, 0xc5, 0x30, 0x63, 0x09, 0x9d, 0x05, 0x1d, 0x46, 0x4e, 0xa7, 0xe5, + 0xc8, 0x51, 0x21, 0x0f, 0x56, 0x08, 0x8a, 0x28, 0x6b, 0x3a, 0x9c, 0xc9, 0xf4, 0x2d, 0xf5, 0xb3, + 0x30, 0xcd, 0xfd, 0x88, 0x4b, 0x30, 0xa3, 0xcf, 0x0e, 0xfa, 0xf5, 0x69, 0x0e, 0x41, 0x50, 0xb0, + 0xa7, 0xfd, 0xae, 0x00, 0xcb, 0x3b, 0x8e, 0xd9, 0x6a, 0x1f, 0x60, 0xd3, 0xeb, 0x58, 0xf6, 0xfe, + 0x55, 0xc7, 0xa6, 0xf8, 0x1e, 0x55, 0xdf, 0x87, 0xb2, 0xdf, 0xc4, 0x99, 0x06, 0x35, 0x44, 0x99, + 0x7d, 0x71, 0x54, 0x66, 0x20, 0x0d, 0x1f, 0xda, 0x6f, 0x62, 0xde, 0xda, 0xfb, 0x0e, 0x6e, 0xd3, + 0x37, 0x31, 0x35, 0x22, 0x13, 0x46, 0x6b, 0x28, 0xa4, 0xaa, 0xbe, 0x0b, 0x25, 0xd2, 0xc3, 0x6d, + 0x91, 0x1c, 0x2f, 0x8d, 0x56, 0x50, 0x9a, 0x8c, 0xad, 0x1e, 0x6e, 0x47, 0x5e, 0xe8, 0x7f, 0x21, + 0x46, 0x51, 0x7d, 0xdf, 0x0f, 0x67, 0x83, 0x7a, 0x84, 0xf5, 0x43, 0xb3, 0x9b, 0x97, 0x27, 0xa0, + 0xcd, 0xf0, 0xf5, 0x8a, 0xa0, 0x7e, 0x92, 0x7f, 0x23, 0x41, 0x57, 0xfb, 0x93, 0x02, 0xd5, 0x34, + 0xb4, 0x37, 0x2c, 0x42, 0xd5, 0x6f, 0x0c, 0xa9, 0xae, 0x31, 0x9e, 0xea, 0x7c, 0x6c, 0xa6, 0xb8, + 0xd0, 0xf1, 0x82, 0x95, 0x98, 0xda, 0xde, 0x81, 0x13, 0x16, 0xc5, 0xdd, 0x20, 0xba, 0x36, 0xf3, + 0x9f, 0x4d, 0x9f, 0x17, 0xe4, 0x4f, 0x6c, 0xfb, 0x84, 0x10, 0xa7, 0xa7, 0x7d, 0x94, 0x71, 0x26, + 0x5f, 0xb1, 0xea, 0x65, 0x98, 0xe3, 0xae, 0x8f, 0x4d, 0xbf, 0xed, 0x14, 0x01, 0xb2, 0x2c, 0x08, + 0xcd, 0xb5, 0x62, 0x7b, 0x48, 0x82, 0x54, 0x5f, 0x85, 0x4a, 0xcf, 0xa1, 0xd8, 0xa6, 0x96, 0xd1, + 0x09, 0x3a, 0x60, 0xdf, 0x1f, 0x59, 0x5b, 0xb8, 0x23, 0xed, 0xa0, 0x04, 0xa4, 0xf6, 0x0b, 0x05, + 0xce, 0x66, 0x5b, 0x47, 0xfd, 0x2e, 0x54, 0x82, 0x13, 0x5f, 0xed, 0x18, 0x56, 0x37, 0x08, 0xb6, + 0x2f, 0x8e, 0xd7, 0x4e, 0x30, 0x9c, 0x88, 0xb6, 0x30, 0xf9, 0x69, 0x71, 0xa6, 0x8a, 0x04, 0x46, + 0x50, 0x82, 0x95, 0xf6, 0xcb, 0x02, 0xcc, 0x4b, 0x20, 0xc7, 0x10, 0x32, 0x6f, 0x4b, 0x21, 0xd3, + 0xcc, 0x73, 0xcc, 0xac, 0x58, 0xb9, 0x95, 0x88, 0x95, 0x0b, 0x79, 0x88, 0x8e, 0x0e, 0x92, 0x81, + 0x02, 0x35, 0x09, 0xfe, 0xaa, 0x63, 0x13, 0xaf, 0xeb, 0xb7, 0xac, 0xb7, 0xb1, 0x8b, 0xfd, 0x8a, + 0xb2, 0x01, 0x65, 0xa3, 0x67, 0x5d, 0x77, 0x1d, 0xaf, 0x97, 0xcc, 0xb9, 0x57, 0x76, 0xb6, 0xd9, + 0x3a, 0x0a, 0x21, 0x7c, 0xe8, 0x40, 0x22, 0x26, 0xed, 0x4c, 0xbc, 0x13, 0x14, 0x2d, 0x62, 0x08, + 0x11, 0x56, 0xab, 0x52, 0x66, 0xb5, 0xd2, 0xa1, 0xe8, 0x59, 0xa6, 0xa8, 0xf9, 0x2f, 0x0a, 0x80, + 0xe2, 0xcd, 0xed, 0xad, 0xff, 0xf4, 0xeb, 0x2f, 0x64, 0x5d, 0x3c, 0xe9, 0xfd, 0x1e, 0x26, 0x8d, + 0x9b, 0xdb, 0x5b, 0xc8, 0x47, 0xd6, 0x3e, 0x56, 0xe0, 0x94, 0x74, 0xc8, 0x63, 0x48, 0x01, 0x3b, + 0x72, 0x0a, 0xf8, 0x42, 0x0e, 0x93, 0x65, 0xc4, 0xfe, 0x4f, 0x8a, 0xb0, 0x2a, 0xc1, 0xc5, 0xda, + 0xf5, 0x27, 0xef, 0xd6, 0x1f, 0xc0, 0x7c, 0x78, 0x7f, 0xbf, 0xe6, 0x3a, 0x5d, 0xe1, 0xdf, 0x5f, + 0xce, 0x71, 0xae, 0xd8, 0x85, 0x23, 0x70, 0x2e, 0xde, 0xf2, 0x5d, 0x8f, 0x13, 0x46, 0x32, 0x9f, + 0xdc, 0x77, 0x67, 0xb5, 0x03, 0x15, 0x53, 0xba, 0x75, 0x55, 0x4b, 0xe3, 0x0c, 0x10, 0xe4, 0x9b, + 0x5a, 0x94, 0x62, 0xe4, 0x75, 0x94, 0xa0, 0xad, 0xfd, 0x4d, 0x81, 0xe7, 0x32, 0x4e, 0x79, 0x0c, + 0x5e, 0xf6, 0x9e, 0xec, 0x65, 0x2f, 0x4f, 0x64, 0x8d, 0x0c, 0x7f, 0xfb, 0xa9, 0x02, 0xeb, 0x47, + 0xd9, 0x2f, 0x67, 0x72, 0x58, 0x87, 0xd2, 0x1d, 0xcb, 0x36, 0x99, 0xef, 0xc4, 0xc2, 0xfd, 0xab, + 0x96, 0x6d, 0x22, 0xb6, 0x13, 0x26, 0x84, 0x62, 0xe6, 0xc5, 0xef, 0x81, 0x02, 0xcf, 0x8f, 0xac, + 0x0e, 0x63, 0xb4, 0xc0, 0x5f, 0x82, 0x05, 0xcf, 0x26, 0x9e, 0x45, 0x7d, 0x87, 0x89, 0x17, 0xbc, + 0xa5, 0x41, 0xbf, 0xbe, 0x70, 0x53, 0xde, 0x42, 0x49, 0x58, 0xed, 0xaf, 0xc9, 0x7c, 0xc2, 0xca, + 0xef, 0x75, 0x38, 0x15, 0x2b, 0x3f, 0x84, 0xc4, 0xae, 0xf8, 0x67, 0x84, 0x0c, 0x71, 0x2c, 0x0e, + 0x80, 0x86, 0x71, 0xfc, 0x50, 0xeb, 0xc5, 0x55, 0xfd, 0xff, 0x0c, 0x35, 0x69, 0x03, 0xc9, 0x7c, + 0xb4, 0x7f, 0x17, 0x60, 0x29, 0xa5, 0x78, 0x4c, 0x34, 0xb5, 0xf8, 0x16, 0x40, 0x34, 0x15, 0x11, + 0x27, 0x68, 0xe4, 0x9b, 0xbd, 0xe8, 0x15, 0x76, 0xb5, 0x88, 0x56, 0x63, 0x14, 0x55, 0x02, 0xb3, + 0x2e, 0x26, 0xd8, 0x3d, 0xc4, 0xe6, 0x35, 0xc7, 0x15, 0x33, 0x8a, 0xd7, 0x72, 0xa8, 0x68, 0xa8, + 0xd0, 0xe9, 0x4b, 0xe2, 0x48, 0xb3, 0x28, 0x22, 0x8c, 0xe2, 0x5c, 0xd4, 0x16, 0xac, 0x98, 0x38, + 0x3e, 0xec, 0x61, 0x49, 0x00, 0x9b, 0xac, 0x7e, 0x95, 0xa3, 0x31, 0xd1, 0x56, 0x1a, 0x10, 0x4a, + 0xc7, 0xd5, 0xfe, 0xa2, 0xc0, 0x8a, 0x24, 0xd9, 0xd7, 0x70, 0xb7, 0xd7, 0x31, 0x28, 0x3e, 0x86, + 0xac, 0x7e, 0x4b, 0x6a, 0x56, 0x5e, 0xc9, 0xa1, 0xbe, 0x40, 0xc8, 0xac, 0xa6, 0x45, 0xfb, 0xb3, + 0x02, 0x67, 0x52, 0x31, 0x8e, 0x21, 0x2d, 0xbe, 0x2b, 0xa7, 0xc5, 0x8b, 0x13, 0x9c, 0x2b, 0x23, + 0x29, 0x3e, 0xca, 0x3a, 0x55, 0x8b, 0x5f, 0x6a, 0x9e, 0xbd, 0xee, 0x52, 0xfb, 0xa4, 0x28, 0x35, + 0xc9, 0xe4, 0x38, 0xba, 0x09, 0x39, 0xa3, 0x14, 0xc6, 0xca, 0x28, 0x43, 0x69, 0xb1, 0x98, 0x33, + 0x2d, 0x12, 0x32, 0x51, 0x5a, 0x54, 0x6f, 0xc1, 0xbc, 0x5c, 0x2b, 0x4a, 0x63, 0x3e, 0x0f, 0x30, + 0xd2, 0x2d, 0xa9, 0x96, 0xc8, 0x94, 0xd4, 0x37, 0x60, 0x99, 0x50, 0xd7, 0x6b, 0x53, 0xcf, 0xc5, + 0x66, 0x6c, 0xbe, 0x7b, 0x82, 0xe5, 0x93, 0xea, 0xa0, 0x5f, 0x5f, 0x6e, 0xa5, 0xec, 0xa3, 0x54, + 0xac, 0x64, 0x9f, 0x4b, 0xc8, 0xd3, 0xdc, 0xe7, 0x92, 0xac, 0xbe, 0xe3, 0x63, 0xb9, 0xcf, 0x8d, + 0x5b, 0xed, 0x59, 0xe8, 0x73, 0x47, 0x78, 0xd9, 0xc8, 0x3e, 0x97, 0xa6, 0x8c, 0xf9, 0x79, 0x55, + 0x3b, 0xa2, 0x6c, 0x26, 0xa7, 0xf9, 0xb9, 0xe6, 0xfc, 0xef, 0xc0, 0xf4, 0x6d, 0x36, 0x81, 0x1c, + 0xb3, 0x4b, 0x0e, 0x0e, 0xca, 0xc7, 0x96, 0xfa, 0x82, 0x60, 0x35, 0xcd, 0xbf, 0x09, 0x0a, 0xa8, + 0x25, 0xfb, 0xe2, 0xb8, 0x56, 0x9e, 0xe6, 0xbe, 0x38, 0x2e, 0x67, 0x86, 0x7f, 0xfe, 0x41, 0xee, + 0x8b, 0x53, 0xed, 0x7d, 0xfc, 0x7d, 0xb1, 0x7f, 0x4f, 0xf2, 0xff, 0x92, 0x9e, 0xd1, 0x0e, 0xee, + 0xd3, 0xe1, 0x3d, 0xe9, 0x46, 0xb0, 0x81, 0x22, 0x18, 0xed, 0x13, 0x05, 0x2a, 0xb2, 0x39, 0x27, + 0x6a, 0xf4, 0x1e, 0x28, 0xb0, 0xe4, 0x4a, 0x64, 0xe2, 0xcf, 0x6d, 0x17, 0xf2, 0xb8, 0x13, 0x7f, + 0x6c, 0x7b, 0x4e, 0x30, 0x5c, 0x4a, 0xd9, 0x44, 0x69, 0xac, 0xb4, 0xef, 0x2b, 0x90, 0x06, 0xac, + 0xda, 0x19, 0x6f, 0xa5, 0x9b, 0x79, 0x06, 0xbd, 0xc2, 0xd3, 0xc7, 0x79, 0x21, 0xfd, 0x7b, 0x4c, + 0xa3, 0xfc, 0x79, 0x79, 0x22, 0x8d, 0xae, 0x43, 0x89, 0x85, 0x45, 0xc2, 0x1b, 0xb6, 0x0c, 0x6a, + 0x20, 0xb6, 0xa3, 0xba, 0x50, 0x89, 0x0a, 0x80, 0xbf, 0xce, 0x0a, 0xc6, 0x91, 0x03, 0xda, 0xa8, + 0x94, 0x24, 0x5e, 0xcb, 0xd9, 0xe1, 0x5a, 0x12, 0x45, 0x94, 0xe0, 0xa0, 0x7d, 0xa8, 0x44, 0x6d, + 0x02, 0x57, 0xef, 0xdd, 0x0c, 0xf5, 0xe6, 0x7a, 0x4c, 0x08, 0x7f, 0x8c, 0xa5, 0xe1, 0x1f, 0x15, + 0x60, 0x21, 0xf1, 0xd2, 0x98, 0xfa, 0x3e, 0xaa, 0x3c, 0xe9, 0xf7, 0xd1, 0xef, 0x29, 0xb0, 0xec, + 0xca, 0x82, 0xc4, 0xdd, 0x7e, 0x33, 0xd7, 0x63, 0x29, 0xf7, 0xfb, 0x35, 0xc1, 0x7e, 0x39, 0x6d, + 0x17, 0xa5, 0x72, 0xd3, 0x7e, 0xa0, 0x40, 0x2a, 0xb8, 0xea, 0x64, 0xd8, 0xe6, 0x62, 0x3e, 0xdb, + 0xf0, 0xb7, 0xdc, 0x71, 0x2c, 0xf3, 0xfb, 0xd8, 0xa8, 0x95, 0xbf, 0x6e, 0x3c, 0xf9, 0x5a, 0xbd, + 0x01, 0x65, 0xdb, 0x31, 0x71, 0xac, 0x87, 0x0c, 0x93, 0xec, 0x0d, 0xb1, 0x8e, 0x42, 0x88, 0x44, + 0x28, 0x16, 0xc7, 0x0a, 0xc5, 0x03, 0x98, 0x77, 0xe3, 0x3e, 0x2f, 0x5a, 0xbf, 0x31, 0xbb, 0x1c, + 0x6e, 0xd7, 0x15, 0xc1, 0x43, 0x8e, 0x1e, 0x24, 0x13, 0x96, 0x7a, 0x37, 0xa6, 0xbf, 0xa7, 0xb6, + 0x77, 0xe3, 0xef, 0xa2, 0xe9, 0xb5, 0xf1, 0xb7, 0x45, 0xa8, 0x66, 0x65, 0x19, 0xf5, 0x43, 0x05, + 0x56, 0x78, 0x20, 0x25, 0xca, 0xe6, 0x64, 0xe1, 0x1a, 0xde, 0xb6, 0x77, 0xd3, 0x68, 0xa2, 0x74, + 0x56, 0xb2, 0x10, 0xf1, 0x41, 0xc9, 0x64, 0xff, 0x53, 0x31, 0x2c, 0x84, 0x34, 0x7c, 0x49, 0x67, + 0x25, 0x39, 0x6e, 0xe9, 0x48, 0xc7, 0xfd, 0x36, 0x4c, 0xbb, 0x6c, 0x20, 0xe2, 0xdf, 0x0b, 0xc6, + 0x78, 0xa8, 0x4c, 0xff, 0x27, 0x9d, 0xa8, 0x57, 0xe3, 0xdf, 0x04, 0x05, 0x54, 0xb5, 0x5f, 0x2b, + 0x30, 0x94, 0xf3, 0x26, 0xaa, 0x5c, 0x06, 0x40, 0xef, 0x7f, 0x54, 0x68, 0xc8, 0x22, 0xa6, 0xc5, + 0x18, 0x51, 0x5d, 0x7f, 0xf8, 0xb8, 0x36, 0xf5, 0xe8, 0x71, 0x6d, 0xea, 0xd3, 0xc7, 0xb5, 0xa9, + 0x07, 0x83, 0x9a, 0xf2, 0x70, 0x50, 0x53, 0x1e, 0x0d, 0x6a, 0xca, 0xa7, 0x83, 0x9a, 0xf2, 0x8f, + 0x41, 0x4d, 0xf9, 0xe8, 0x9f, 0xb5, 0xa9, 0xf7, 0xd6, 0x46, 0xfd, 0x3b, 0xdf, 0x7f, 0x03, 0x00, + 0x00, 0xff, 0xff, 0xf0, 0x19, 0xbb, 0x6c, 0xed, 0x27, 0x00, 0x00, } func (m *AllocationResult) Marshal() (dAtA []byte, err error) { @@ -2539,11 +2537,6 @@ func (m *ResourceClaimSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - i -= len(m.AllocationMode) - copy(dAtA[i:], m.AllocationMode) - i = encodeVarintGenerated(dAtA, i, uint64(len(m.AllocationMode))) - i-- - dAtA[i] = 0x1a if m.ParametersRef != nil { { size, err := m.ParametersRef.MarshalToSizedBuffer(dAtA[:i]) @@ -3886,8 +3879,6 @@ func (m *ResourceClaimSpec) Size() (n int) { l = m.ParametersRef.Size() n += 1 + l + sovGenerated(uint64(l)) } - l = len(m.AllocationMode) - n += 1 + l + sovGenerated(uint64(l)) return n } @@ -4536,7 +4527,6 @@ func (this *ResourceClaimSpec) String() string { s := strings.Join([]string{`&ResourceClaimSpec{`, `ResourceClassName:` + fmt.Sprintf("%v", this.ResourceClassName) + `,`, `ParametersRef:` + strings.Replace(this.ParametersRef.String(), "ResourceClaimParametersReference", "ResourceClaimParametersReference", 1) + `,`, - `AllocationMode:` + fmt.Sprintf("%v", this.AllocationMode) + `,`, `}`, }, "") return s @@ -7889,38 +7879,6 @@ func (m *ResourceClaimSpec) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AllocationMode", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthGenerated - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.AllocationMode = AllocationMode(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/resource/v1alpha3/generated.proto b/staging/src/k8s.io/api/resource/v1alpha3/generated.proto index 231c4735970..56045f35dbf 100644 --- a/staging/src/k8s.io/api/resource/v1alpha3/generated.proto +++ b/staging/src/k8s.io/api/resource/v1alpha3/generated.proto @@ -411,11 +411,6 @@ message ResourceClaimSpec { // The object must be in the same namespace as the ResourceClaim. // +optional optional ResourceClaimParametersReference parametersRef = 2; - - // Allocation can start immediately or when a Pod wants to use the - // resource. "WaitForFirstConsumer" is the default. - // +optional - optional string allocationMode = 3; } // ResourceClaimStatus tracks whether the resource has been allocated and what diff --git a/staging/src/k8s.io/api/resource/v1alpha3/types.go b/staging/src/k8s.io/api/resource/v1alpha3/types.go index 1bc10886d42..6f2323a8440 100644 --- a/staging/src/k8s.io/api/resource/v1alpha3/types.go +++ b/staging/src/k8s.io/api/resource/v1alpha3/types.go @@ -72,34 +72,8 @@ type ResourceClaimSpec struct { // The object must be in the same namespace as the ResourceClaim. // +optional ParametersRef *ResourceClaimParametersReference `json:"parametersRef,omitempty" protobuf:"bytes,2,opt,name=parametersRef"` - - // Allocation can start immediately or when a Pod wants to use the - // resource. "WaitForFirstConsumer" is the default. - // +optional - AllocationMode AllocationMode `json:"allocationMode,omitempty" protobuf:"bytes,3,opt,name=allocationMode"` } -// AllocationMode describes whether a ResourceClaim gets allocated immediately -// when it gets created (AllocationModeImmediate) or whether allocation is -// delayed until it is needed for a Pod -// (AllocationModeWaitForFirstConsumer). Other modes might get added in the -// future. -type AllocationMode string - -const ( - // When a ResourceClaim has AllocationModeWaitForFirstConsumer, allocation is - // delayed until a Pod gets scheduled that needs the ResourceClaim. The - // scheduler will consider all resource requirements of that Pod and - // trigger allocation for a node that fits the Pod. - AllocationModeWaitForFirstConsumer AllocationMode = "WaitForFirstConsumer" - - // When a ResourceClaim has AllocationModeImmediate, allocation starts - // as soon as the ResourceClaim gets created. This is done without - // considering the needs of Pods that will use the ResourceClaim - // because those Pods are not known yet. - AllocationModeImmediate AllocationMode = "Immediate" -) - // ResourceClaimStatus tracks whether the resource has been allocated and what // the resulting attributes are. type ResourceClaimStatus struct { diff --git a/staging/src/k8s.io/api/resource/v1alpha3/types_swagger_doc_generated.go b/staging/src/k8s.io/api/resource/v1alpha3/types_swagger_doc_generated.go index 2aaf4d63a97..7a1e6457b43 100644 --- a/staging/src/k8s.io/api/resource/v1alpha3/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/resource/v1alpha3/types_swagger_doc_generated.go @@ -187,7 +187,6 @@ var map_ResourceClaimSpec = map[string]string{ "": "ResourceClaimSpec defines how a resource is to be allocated.", "resourceClassName": "ResourceClassName references the driver and additional parameters via the name of a ResourceClass that was created as part of the driver deployment.", "parametersRef": "ParametersRef references a separate object with arbitrary parameters that will be used by the driver when allocating a resource for the claim.\n\nThe object must be in the same namespace as the ResourceClaim.", - "allocationMode": "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.", } func (ResourceClaimSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.json b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.json index b364278a21f..b58b8a72dbb 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.json +++ b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.json @@ -49,8 +49,7 @@ "apiGroup": "apiGroupValue", "kind": "kindValue", "name": "nameValue" - }, - "allocationMode": "allocationModeValue" + } }, "status": { "driverName": "driverNameValue", diff --git a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.pb b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.pb index 374ea128cb0b36e230afa14307dc528b2b344b54..880bfadef71eb3aa5d72751b76600422b7b3016b 100644 GIT binary patch delta 23 fcmZqY_{2WJfN{@8!&F8_`^nQ89X7vVlx6||V!j8q delta 38 ucmeyw-p(<>fbrr+!&FAbkjc{-9r%P3b8_;N6H7Al^L+DDHeX;=VgdjIbqz8A diff --git a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.yaml b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.yaml index 27144e49a06..938aad50862 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaim.yaml @@ -33,7 +33,6 @@ metadata: selfLink: selfLinkValue uid: uidValue spec: - allocationMode: allocationModeValue parametersRef: apiGroup: apiGroupValue kind: kindValue diff --git a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaimTemplate.json b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaimTemplate.json index 9d31265a7b6..45a991029ef 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaimTemplate.json +++ b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaimTemplate.json @@ -92,8 +92,7 @@ "apiGroup": "apiGroupValue", "kind": "kindValue", "name": "nameValue" - }, - "allocationMode": "allocationModeValue" + } } } } \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaimTemplate.pb b/staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha3.ResourceClaimTemplate.pb index 0396e3404d652ec0b06b919739066cff898e3674..2d8b866d6be7d055562ccebb6edb486b1d3eb37d 100644 GIT binary patch delta 31 ncmcc1c7kn!1ydK><^_x?jEqY*&u2_!WVD~m%Iv_*pu_+Gq<9Ep delta 52 zcmX@Xc9(5}1=D=C%?lV)7#VkNp3j)d$QUx2mDxc;I58(DKRK}^Ge6HaKP5FRF{d remove finalizer - "immediate-deleted-finalizer-removal": { - key: claimKey, - classes: classes, - claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), - driver: m.expectDeallocate(map[string]error{claimName: nil}), - expectedClaim: withDeletionTimestamp(claim), - }, - // deletion time stamp set, our finalizer set, not allocated, stopping fails -> requeue - "immediate-deleted-finalizer-stop-failure": { - key: claimKey, - classes: classes, - claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), - driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), - expectedClaim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), - expectedError: "stop allocation: fake error", - }, - // deletion time stamp set, other finalizer set, not allocated -> do nothing - "immediate-deleted-finalizer-no-removal": { - key: claimKey, - classes: classes, - claim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer), - expectedClaim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer), - }, - // deletion time stamp set, finalizer set, allocated -> deallocate - "immediate-deleted-allocated": { - key: claimKey, - classes: classes, - claim: withAllocate(withDeletionTimestamp(claim)), - driver: m.expectDeallocate(map[string]error{claimName: nil}), - expectedClaim: withDeletionTimestamp(claim), - }, - // deletion time stamp set, finalizer set, allocated, deallocation fails -> requeue - "immediate-deleted-deallocate-failure": { - key: claimKey, - classes: classes, - claim: withAllocate(withDeletionTimestamp(claim)), - driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), - expectedClaim: withAllocate(withDeletionTimestamp(claim)), - expectedError: "deallocate: fake error", - }, - // deletion time stamp set, finalizer not set -> do nothing - "immediate-deleted-no-finalizer": { - key: claimKey, - classes: classes, - claim: withDeletionTimestamp(claim), - expectedClaim: withDeletionTimestamp(claim), - }, - // not deleted, not allocated, no finalizer -> add finalizer, allocate - "immediate-do-allocation": { - key: claimKey, - classes: classes, - claim: claim, - driver: m.expectClassParameters(map[string]interface{}{className: 1}). - expectClaimParameters(map[string]interface{}{claimName: 2}). - expectAllocate(map[string]allocate{claimName: {allocResult: &allocation, allocErr: nil}}), - expectedClaim: withAllocate(claim), - }, - // not deleted, not allocated, finalizer -> allocate - "immediate-continue-allocation": { - key: claimKey, - classes: classes, - claim: withFinalizer(claim, ourFinalizer), - driver: m.expectClassParameters(map[string]interface{}{className: 1}). - expectClaimParameters(map[string]interface{}{claimName: 2}). - expectAllocate(map[string]allocate{claimName: {allocResult: &allocation, allocErr: nil}}), - expectedClaim: withAllocate(claim), - }, - // not deleted, not allocated, finalizer, fail allocation -> requeue - "immediate-fail-allocation": { - key: claimKey, - classes: classes, - claim: withFinalizer(claim, ourFinalizer), - driver: m.expectClassParameters(map[string]interface{}{className: 1}). - expectClaimParameters(map[string]interface{}{claimName: 2}). - expectAllocate(map[string]allocate{claimName: {allocErr: errors.New("fake error")}}), - expectedClaim: withFinalizer(claim, ourFinalizer), - expectedError: "allocate: fake error", - }, - // not deleted, allocated -> do nothing - "immediate-allocated-nop": { - key: claimKey, - classes: classes, - claim: withAllocate(claim), - expectedClaim: withAllocate(claim), }, // not deleted, reallocate -> deallocate @@ -257,62 +167,60 @@ func TestController(t *testing.T) { expectedError: "deallocate: fake error", }, - // Delayed allocation is similar in some cases, but not quite - // the same. // deletion time stamp set, our finalizer set, not allocated -> remove finalizer - "delayed-deleted-finalizer-removal": { + "deleted-finalizer-removal": { key: claimKey, classes: classes, - claim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer), + claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), driver: m.expectDeallocate(map[string]error{claimName: nil}), - expectedClaim: withDeletionTimestamp(delayedClaim), + expectedClaim: withDeletionTimestamp(claim), }, // deletion time stamp set, our finalizer set, not allocated, stopping fails -> requeue - "delayed-deleted-finalizer-stop-failure": { + "deleted-finalizer-stop-failure": { key: claimKey, classes: classes, - claim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer), + claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), - expectedClaim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer), + expectedClaim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), expectedError: "stop allocation: fake error", }, // deletion time stamp set, other finalizer set, not allocated -> do nothing - "delayed-deleted-finalizer-no-removal": { + "deleted-finalizer-no-removal": { key: claimKey, classes: classes, - claim: withFinalizer(withDeletionTimestamp(delayedClaim), otherFinalizer), - expectedClaim: withFinalizer(withDeletionTimestamp(delayedClaim), otherFinalizer), + claim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer), + expectedClaim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer), }, // deletion time stamp set, finalizer set, allocated -> deallocate - "delayed-deleted-allocated": { + "deleted-allocated": { key: claimKey, classes: classes, - claim: withAllocate(withDeletionTimestamp(delayedClaim)), + claim: withAllocate(withDeletionTimestamp(claim)), driver: m.expectDeallocate(map[string]error{claimName: nil}), - expectedClaim: withDeletionTimestamp(delayedClaim), + expectedClaim: withDeletionTimestamp(claim), }, // deletion time stamp set, finalizer set, allocated, deallocation fails -> requeue - "delayed-deleted-deallocate-failure": { + "deleted-deallocate-failure": { key: claimKey, classes: classes, - claim: withAllocate(withDeletionTimestamp(delayedClaim)), + claim: withAllocate(withDeletionTimestamp(claim)), driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), - expectedClaim: withAllocate(withDeletionTimestamp(delayedClaim)), + expectedClaim: withAllocate(withDeletionTimestamp(claim)), expectedError: "deallocate: fake error", }, // deletion time stamp set, finalizer not set -> do nothing - "delayed-deleted-no-finalizer": { + "deleted-no-finalizer": { key: claimKey, classes: classes, - claim: withDeletionTimestamp(delayedClaim), - expectedClaim: withDeletionTimestamp(delayedClaim), + claim: withDeletionTimestamp(claim), + expectedClaim: withDeletionTimestamp(claim), }, // waiting for first consumer -> do nothing - "delayed-pending": { + "pending": { key: claimKey, classes: classes, - claim: delayedClaim, - expectedClaim: delayedClaim, + claim: claim, + expectedClaim: claim, }, // pod with no claims -> shouldn't occur, check again anyway @@ -324,34 +232,23 @@ func TestController(t *testing.T) { expectedError: errPeriodic.Error(), }, - // pod with immediate allocation and selected node -> shouldn't occur, check again in case that claim changes - "pod-immediate": { - key: podKey, - claim: claim, - expectedClaim: claim, - pod: podWithClaim, - schedulingCtx: withSelectedNode(podSchedulingCtx), - expectedSchedulingCtx: withSelectedNode(podSchedulingCtx), - expectedError: errPeriodic.Error(), - }, - - // pod with delayed allocation, no potential nodes -> shouldn't occur - "pod-delayed-no-nodes": { + // no potential nodes -> shouldn't occur + "no-nodes": { key: podKey, classes: classes, - claim: delayedClaim, - expectedClaim: delayedClaim, + claim: claim, + expectedClaim: claim, pod: podWithClaim, schedulingCtx: podSchedulingCtx, expectedSchedulingCtx: podSchedulingCtx, }, - // pod with delayed allocation, potential nodes -> provide unsuitable nodes - "pod-delayed-info": { + // potential nodes -> provide unsuitable nodes + "info": { key: podKey, classes: classes, - claim: delayedClaim, - expectedClaim: delayedClaim, + claim: claim, + expectedClaim: claim, pod: podWithClaim, schedulingCtx: withPotentialNodes(podSchedulingCtx), driver: m.expectClassParameters(map[string]interface{}{className: 1}). @@ -361,23 +258,23 @@ func TestController(t *testing.T) { expectedError: errPeriodic.Error(), }, - // pod with delayed allocation, potential nodes, selected node, missing class -> failure - "pod-delayed-missing-class": { + // potential nodes, selected node, missing class -> failure + "missing-class": { key: podKey, - claim: delayedClaim, - expectedClaim: delayedClaim, + claim: claim, + expectedClaim: claim, pod: podWithClaim, schedulingCtx: withSelectedNode(withPotentialNodes(podSchedulingCtx)), expectedSchedulingCtx: withSelectedNode(withPotentialNodes(podSchedulingCtx)), expectedError: `pod claim my-pod-claim: resourceclass.resource.k8s.io "mock-class" not found`, }, - // pod with delayed allocation, potential nodes, selected node -> allocate - "pod-delayed-allocate": { + // potential nodes, selected node -> allocate + "allocate": { key: podKey, classes: classes, - claim: delayedClaim, - expectedClaim: withReservedFor(withAllocate(delayedClaim), pod), + claim: claim, + expectedClaim: withReservedFor(withAllocate(claim), pod), pod: podWithClaim, schedulingCtx: withSelectedNode(withPotentialNodes(podSchedulingCtx)), driver: m.expectClassParameters(map[string]interface{}{className: 1}). @@ -387,12 +284,12 @@ func TestController(t *testing.T) { expectedSchedulingCtx: withUnsuitableNodes(withSelectedNode(withPotentialNodes(podSchedulingCtx))), expectedError: errPeriodic.Error(), }, - // pod with delayed allocation, potential nodes, selected node, all unsuitable -> update unsuitable nodes - "pod-selected-is-potential-node": { + // potential nodes, selected node, all unsuitable -> update unsuitable nodes + "is-potential-node": { key: podKey, classes: classes, - claim: delayedClaim, - expectedClaim: delayedClaim, + claim: claim, + expectedClaim: claim, pod: podWithClaim, schedulingCtx: withPotentialNodes(withSelectedNode(withPotentialNodes(podSchedulingCtx))), driver: m.expectClassParameters(map[string]interface{}{className: 1}). @@ -401,12 +298,12 @@ func TestController(t *testing.T) { expectedSchedulingCtx: withSpecificUnsuitableNodes(withSelectedNode(withPotentialNodes(podSchedulingCtx)), potentialNodes), expectedError: errPeriodic.Error(), }, - // pod with delayed allocation, max potential nodes, other selected node, all unsuitable -> update unsuitable nodes with truncation at start - "pod-selected-is-potential-node-truncate-first": { + // max potential nodes, other selected node, all unsuitable -> update unsuitable nodes with truncation at start + "is-potential-node-truncate-first": { key: podKey, classes: classes, - claim: delayedClaim, - expectedClaim: delayedClaim, + claim: claim, + expectedClaim: claim, pod: podWithClaim, schedulingCtx: withSpecificPotentialNodes(withSelectedNode(withSpecificPotentialNodes(podSchedulingCtx, maxNodes)), maxNodes), driver: m.expectClassParameters(map[string]interface{}{className: 1}). @@ -415,12 +312,12 @@ func TestController(t *testing.T) { expectedSchedulingCtx: withSpecificUnsuitableNodes(withSelectedNode(withSpecificPotentialNodes(podSchedulingCtx, maxNodes)), append(maxNodes[1:], nodeName)), expectedError: errPeriodic.Error(), }, - // pod with delayed allocation, max potential nodes, other selected node, all unsuitable (but in reverse order) -> update unsuitable nodes with truncation at end + // max potential nodes, other selected node, all unsuitable (but in reverse order) -> update unsuitable nodes with truncation at end "pod-selected-is-potential-node-truncate-last": { key: podKey, classes: classes, - claim: delayedClaim, - expectedClaim: delayedClaim, + claim: claim, + expectedClaim: claim, pod: podWithClaim, schedulingCtx: withSpecificPotentialNodes(withSelectedNode(withSpecificPotentialNodes(podSchedulingCtx, maxNodes)), maxNodes), driver: m.expectClassParameters(map[string]interface{}{className: 1}). @@ -652,7 +549,6 @@ func createClaim(claimName, claimNamespace, className string) *resourceapi.Resou }, Spec: resourceapi.ResourceClaimSpec{ ResourceClassName: className, - AllocationMode: resourceapi.AllocationModeImmediate, }, } } diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index 5e76efc589e..7b06cbdf891 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -27,7 +27,6 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - "github.com/onsi/gomega/gcustom" "github.com/onsi/gomega/gstruct" "github.com/onsi/gomega/types" @@ -100,7 +99,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.By("waiting for container startup to fail") parameters := b.parameters() - pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, template := b.podInline() b.create(ctx, parameters, pod, template) @@ -126,7 +125,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // Pretend that the resource is allocated and reserved for some other entity. // Until the resourceclaim controller learns to remove reservations for // arbitrary types we can simply fake somthing here. - claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + claim := b.externalClaim() b.create(ctx, claim) claim, err := f.ClientSet.ResourceV1alpha3().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) @@ -180,7 +179,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("must unprepare resources for force-deleted pod", func(ctx context.Context) { parameters := b.parameters() - claim := b.externalClaim(resourceapi.AllocationModeImmediate) + claim := b.externalClaim() pod := b.podExternal() zero := int64(0) pod.Spec.TerminationGracePeriodSeconds = &zero @@ -203,7 +202,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("must skip NodePrepareResource if not used by any container", func(ctx context.Context) { parameters := b.parameters() - pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, template := b.podInline() for i := range pod.Spec.Containers { pod.Spec.Containers[i].Resources.Claims = nil } @@ -219,10 +218,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // claimTests tries out several different combinations of pods with // claims, both inline and external. - claimTests := func(b *builder, driver *Driver, allocationMode resourceapi.AllocationMode) { + claimTests := func(b *builder, driver *Driver) { ginkgo.It("supports simple pod referencing inline resource claim", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() - pod, template := b.podInline(allocationMode) + pod, template := b.podInline() objects = append(objects, pod, template) b.create(ctx, objects...) @@ -231,7 +230,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("supports inline claim referenced by multiple containers", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() - pod, template := b.podInlineMultiple(allocationMode) + pod, template := b.podInlineMultiple() objects = append(objects, pod, template) b.create(ctx, objects...) @@ -241,7 +240,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() pod := b.podExternal() - claim := b.externalClaim(allocationMode) + claim := b.externalClaim() objects = append(objects, claim, pod) b.create(ctx, objects...) @@ -253,7 +252,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, pod1 := b.podExternal() pod2 := b.podExternal() pod3 := b.podExternal() - claim := b.externalClaim(allocationMode) + claim := b.externalClaim() objects = append(objects, claim, pod1, pod2, pod3) b.create(ctx, objects...) @@ -267,7 +266,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, pod1 := b.podExternalMultiple() pod2 := b.podExternalMultiple() pod3 := b.podExternalMultiple() - claim := b.externalClaim(allocationMode) + claim := b.externalClaim() objects = append(objects, claim, pod1, pod2, pod3) b.create(ctx, objects...) @@ -278,7 +277,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("supports init containers", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() - pod, template := b.podInline(allocationMode) + pod, template := b.podInline() pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]} pod.Spec.InitContainers[0].Name += "-init" // This must succeed for the pod to start. @@ -292,7 +291,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) { objects, _ := b.flexibleParameters() pod := b.podExternal() - claim := b.externalClaim(allocationMode) + claim := b.externalClaim() pod.Spec.Containers[0].Command = []string{"true"} objects = append(objects, claim, pod) b.create(ctx, objects...) @@ -307,7 +306,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("deletes generated claims when pod is done", func(ctx context.Context) { objects, _ := b.flexibleParameters() - pod, template := b.podInline(allocationMode) + pod, template := b.podInline() pod.Spec.Containers[0].Command = []string{"true"} objects = append(objects, template, pod) b.create(ctx, objects...) @@ -326,7 +325,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("does not delete generated claims when pod is restarting", func(ctx context.Context) { objects, _ := b.flexibleParameters() - pod, template := b.podInline(allocationMode) + pod, template := b.podInline() pod.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 1; exit 1"} pod.Spec.RestartPolicy = v1.RestartPolicyAlways objects = append(objects, template, pod) @@ -341,10 +340,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, } }) - ginkgo.It("must deallocate after use when using delayed allocation", func(ctx context.Context) { + ginkgo.It("must deallocate after use", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() pod := b.podExternal() - claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + claim := b.externalClaim() objects = append(objects, claim, pod) b.create(ctx, objects...) @@ -383,7 +382,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("supports claim and class parameters", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() - pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, template := b.podInline() objects = append(objects, pod, template) b.create(ctx, objects...) @@ -395,7 +394,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, objects, expectedEnv := b.flexibleParameters() pods := make([]*v1.Pod, numPods) for i := 0; i < numPods; i++ { - pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, template := b.podInline() pods[i] = pod objects = append(objects, pod, template) } @@ -421,7 +420,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() - objects = append(objects, b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)) + objects = append(objects, b.externalClaim()) pods := make([]*v1.Pod, numPods) for i := 0; i < numPods; i++ { @@ -459,7 +458,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, objects[len(objects)-1].(*resourceapi.ResourceClaimParameters).Shareable = false } - objects = append(objects, b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)) + objects = append(objects, b.externalClaim()) pods := make([]*v1.Pod, numPods) for i := 0; i < numPods; i++ { @@ -491,7 +490,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("retries pod scheduling after creating resource class", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() - pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, template := b.podInline() class, err := f.ClientSet.ResourceV1alpha3().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{}) framework.ExpectNoError(err) template.Spec.Spec.ResourceClassName += "-b" @@ -510,7 +509,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("retries pod scheduling after updating resource class", func(ctx context.Context) { objects, expectedEnv := b.flexibleParameters() - pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, template := b.podInline() // First modify the class so that it matches no nodes. class, err := f.ClientSet.ResourceV1alpha3().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{}) @@ -546,7 +545,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) ginkgo.It("runs a pod without a generated resource claim", func(ctx context.Context) { - pod, _ /* template */ := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, _ /* template */ := b.podInline() created := b.create(ctx, pod) pod = created[0].(*v1.Pod) @@ -563,13 +562,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)) }) - ginkgo.Context("with delayed allocation", func() { - claimTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer) - }) - - ginkgo.Context("with immediate allocation", func() { - claimTests(b, driver, resourceapi.AllocationModeImmediate) - }) + claimTests(b, driver) } // These tests depend on having more than one node and a DRA driver controller. @@ -603,7 +596,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, } pod1 := createPod() pod2 := createPod() - claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + claim := b.externalClaim() b.create(ctx, parameters, claim, pod1, pod2) for _, pod := range []*v1.Pod{pod1, pod2} { @@ -624,7 +617,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, instance := f.UniqueName + "-test-app" pod := b.podExternal() pod.Labels[label] = instance - claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + claim := b.externalClaim() b.create(ctx, parameters, claim, pod) ginkgo.By("wait for test pod " + pod.Name + " to run") @@ -714,13 +707,13 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, parameters1 := b.parameters() parameters2 := b2.parameters() // Order is relevant here: each pod must be matched with its own claim. - pod1claim1 := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + pod1claim1 := b.externalClaim() pod1 := b.podExternal() - pod2claim1 := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + pod2claim1 := b2.externalClaim() pod2 := b2.podExternal() // Add another claim to pod1. - pod1claim2 := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + pod1claim2 := b2.externalClaim() pod1.Spec.ResourceClaims = append(pod1.Spec.ResourceClaims, v1.PodResourceClaim{ Name: "claim-other", @@ -801,51 +794,41 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, driver.parameterMode = parameterMode b := newBuilder(f, driver) - tests := func(allocationMode resourceapi.AllocationMode) { - ginkgo.It("uses all resources", func(ctx context.Context) { - objs, _ := b.flexibleParameters() - var pods []*v1.Pod - for i := 0; i < len(nodes.NodeNames); i++ { - pod, template := b.podInline(allocationMode) - pods = append(pods, pod) - objs = append(objs, pod, template) + ginkgo.It("uses all resources", func(ctx context.Context) { + objs, _ := b.flexibleParameters() + var pods []*v1.Pod + for i := 0; i < len(nodes.NodeNames); i++ { + pod, template := b.podInline() + pods = append(pods, pod) + objs = append(objs, pod, template) + } + b.create(ctx, objs...) + + for _, pod := range pods { + err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) + framework.ExpectNoError(err, "start pod") + } + + // The pods all should run on different + // nodes because the maximum number of + // claims per node was limited to 1 for + // this test. + // + // We cannot know for sure why the pods + // ran on two different nodes (could + // also be a coincidence) but if they + // don't cover all nodes, then we have + // a problem. + used := make(map[string]*v1.Pod) + for _, pod := range pods { + pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "get pod") + nodeName := pod.Spec.NodeName + if other, ok := used[nodeName]; ok { + framework.Failf("Pod %s got started on the same node %s as pod %s although claim allocation should have been limited to one claim per node.", pod.Name, nodeName, other.Name) } - b.create(ctx, objs...) - - for _, pod := range pods { - err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) - framework.ExpectNoError(err, "start pod") - } - - // The pods all should run on different - // nodes because the maximum number of - // claims per node was limited to 1 for - // this test. - // - // We cannot know for sure why the pods - // ran on two different nodes (could - // also be a coincidence) but if they - // don't cover all nodes, then we have - // a problem. - used := make(map[string]*v1.Pod) - for _, pod := range pods { - pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) - framework.ExpectNoError(err, "get pod") - nodeName := pod.Spec.NodeName - if other, ok := used[nodeName]; ok { - framework.Failf("Pod %s got started on the same node %s as pod %s although claim allocation should have been limited to one claim per node.", pod.Name, nodeName, other.Name) - } - used[nodeName] = pod - } - }) - } - - ginkgo.Context("with delayed allocation", func() { - tests(resourceapi.AllocationModeWaitForFirstConsumer) - }) - - ginkgo.Context("with immediate allocation", func() { - tests(resourceapi.AllocationModeImmediate) + used[nodeName] = pod + } }) }) } @@ -873,7 +856,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("truncates the name of a generated resource claim", func(ctx context.Context) { parameters := b.parameters() - pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) + pod, template := b.podInline() pod.Name = strings.Repeat("p", 63) pod.Spec.ResourceClaims[0].Name = strings.Repeat("c", 63) pod.Spec.Containers[0].Resources.Claims[0].Name = pod.Spec.ResourceClaims[0].Name @@ -1039,118 +1022,16 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // which is the goal for 1.31 to support version skew for kubelet. }) - ginkgo.Context("with local unshared resources", func() { - driver := NewDriver(f, nodes, func() app.Resources { - return app.Resources{ - NodeLocal: true, - MaxAllocations: 10, - Nodes: nodes.NodeNames, - } - }) - b := newBuilder(f, driver) - - // This test covers some special code paths in the scheduler: - // - Patching the ReservedFor during PreBind because in contrast - // to claims specifically allocated for a pod, here the claim - // gets allocated without reserving it. - // - Error handling when PreBind fails: multiple attempts to bind pods - // are started concurrently, only one attempt succeeds. - // - Removing a ReservedFor entry because the first inline claim gets - // reserved during allocation. - ginkgo.It("reuses an allocated immediate claim", func(ctx context.Context) { - objects := []klog.KMetadata{ - b.parameters(), - b.externalClaim(resourceapi.AllocationModeImmediate), - } - podExternal := b.podExternal() - - // Create many pods to increase the chance that the scheduler will - // try to bind two pods at the same time. - numPods := 5 - for i := 0; i < numPods; i++ { - podInline, claimTemplate := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer) - podInline.Spec.Containers[0].Resources.Claims = append(podInline.Spec.Containers[0].Resources.Claims, podExternal.Spec.Containers[0].Resources.Claims[0]) - podInline.Spec.ResourceClaims = append(podInline.Spec.ResourceClaims, podExternal.Spec.ResourceClaims[0]) - objects = append(objects, claimTemplate, podInline) - } - b.create(ctx, objects...) - - var runningPod *v1.Pod - haveRunningPod := gcustom.MakeMatcher(func(pods []v1.Pod) (bool, error) { - numRunning := 0 - runningPod = nil - for _, pod := range pods { - if pod.Status.Phase == v1.PodRunning { - pod := pod // Don't keep pointer to loop variable... - runningPod = &pod - numRunning++ - } - } - return numRunning == 1, nil - }).WithTemplate("Expected one running Pod.\nGot instead:\n{{.FormattedActual}}") - - for i := 0; i < numPods; i++ { - ginkgo.By("waiting for exactly one pod to start") - runningPod = nil - gomega.Eventually(ctx, b.listTestPods).WithTimeout(f.Timeouts.PodStartSlow).Should(haveRunningPod) - - ginkgo.By("checking that no other pod gets scheduled") - havePendingPods := gcustom.MakeMatcher(func(pods []v1.Pod) (bool, error) { - numPending := 0 - for _, pod := range pods { - if pod.Status.Phase == v1.PodPending { - numPending++ - } - } - return numPending == numPods-1-i, nil - }).WithTemplate("Expected only one running Pod.\nGot instead:\n{{.FormattedActual}}") - gomega.Consistently(ctx, b.listTestPods).WithTimeout(time.Second).Should(havePendingPods) - - ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(runningPod))) - framework.ExpectNoError(b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, runningPod.Name, metav1.DeleteOptions{})) - - ginkgo.By(fmt.Sprintf("waiting for pod %s to disappear", klog.KObj(runningPod))) - framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, b.f.ClientSet, runningPod.Name, runningPod.Namespace, f.Timeouts.PodDelete)) - } - }) - }) - - ginkgo.Context("with shared network resources", func() { - driver := NewDriver(f, nodes, networkResources) - b := newBuilder(f, driver) - - // This test complements "reuses an allocated immediate claim" above: - // because the claim can be shared, each PreBind attempt succeeds. - ginkgo.It("shares an allocated immediate claim", func(ctx context.Context) { - objects := []klog.KMetadata{ - b.parameters(), - b.externalClaim(resourceapi.AllocationModeImmediate), - } - // Create many pods to increase the chance that the scheduler will - // try to bind two pods at the same time. - numPods := 5 - pods := make([]*v1.Pod, numPods) - for i := 0; i < numPods; i++ { - pods[i] = b.podExternal() - objects = append(objects, pods[i]) - } - b.create(ctx, objects...) - - ginkgo.By("waiting all pods to start") - framework.ExpectNoError(e2epod.WaitForPodsRunning(ctx, b.f.ClientSet, f.Namespace.Name, numPods+len(nodes.NodeNames) /* driver(s) */, f.Timeouts.PodStartSlow)) - }) - }) - // kube-controller-manager can trigger delayed allocation for pods where the // node name was already selected when creating the pod. For immediate // allocation, the creator has to ensure that the node matches the claims. // This does not work for resource claim templates and only isn't // a problem here because the resource is network-attached and available // on all nodes. - preScheduledTests := func(b *builder, driver *Driver, allocationMode resourceapi.AllocationMode) { + preScheduledTests := func(b *builder, driver *Driver) { ginkgo.It("supports scheduled pod referencing inline resource claim", func(ctx context.Context) { parameters := b.parameters() - pod, template := b.podInline(allocationMode) + pod, template := b.podInline() pod.Spec.NodeName = nodes.NodeNames[0] b.create(ctx, parameters, pod, template) @@ -1159,7 +1040,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("supports scheduled pod referencing external resource claim", func(ctx context.Context) { parameters := b.parameters() - claim := b.externalClaim(allocationMode) + claim := b.externalClaim() pod := b.podExternal() pod.Spec.NodeName = nodes.NodeNames[0] b.create(ctx, parameters, claim, pod) @@ -1168,30 +1049,22 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) } - ginkgo.Context("with delayed allocation and setting ReservedFor", func() { + ginkgo.Context("with setting ReservedFor", func() { driver := NewDriver(f, nodes, networkResources) b := newBuilder(f, driver) - preScheduledTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer) - claimTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer) + preScheduledTests(b, driver) + claimTests(b, driver) }) - ginkgo.Context("with delayed allocation and not setting ReservedFor", func() { + ginkgo.Context("without setting ReservedFor", func() { driver := NewDriver(f, nodes, func() app.Resources { resources := networkResources() resources.DontSetReservedFor = true return resources }) b := newBuilder(f, driver) - preScheduledTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer) - claimTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer) - }) - - ginkgo.Context("with immediate allocation", func() { - driver := NewDriver(f, nodes, networkResources) - b := newBuilder(f, driver) - preScheduledTests(b, driver, resourceapi.AllocationModeImmediate) - claimTests(b, driver, resourceapi.AllocationModeImmediate) - + preScheduledTests(b, driver) + claimTests(b, driver) }) }) @@ -1209,10 +1082,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("work", func(ctx context.Context) { parameters1 := b1.parameters() parameters2 := b2.parameters() - claim1 := b1.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) - claim1b := b1.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) - claim2 := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) - claim2b := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer) + claim1 := b1.externalClaim() + claim1b := b1.externalClaim() + claim2 := b2.externalClaim() + claim2b := b2.externalClaim() pod := b1.podExternal() for i, claim := range []*resourceapi.ResourceClaim{claim1b, claim2, claim2b} { claim := claim @@ -1298,7 +1171,7 @@ func (b *builder) nodeSelector() *v1.NodeSelector { // externalClaim returns external resource claim // that test pods can reference -func (b *builder) externalClaim(allocationMode resourceapi.AllocationMode) *resourceapi.ResourceClaim { +func (b *builder) externalClaim() *resourceapi.ResourceClaim { b.claimCounter++ name := "external-claim" + b.driver.NameSuffix // This is what podExternal expects. if b.claimCounter > 1 { @@ -1315,7 +1188,6 @@ func (b *builder) externalClaim(allocationMode resourceapi.AllocationMode) *reso Kind: b.driver.claimParameterAPIKind, Name: b.parametersName(), }, - AllocationMode: allocationMode, }, } } @@ -1493,7 +1365,7 @@ func (b *builder) pod() *v1.Pod { } // makePodInline adds an inline resource claim with default class name and parameters. -func (b *builder) podInline(allocationMode resourceapi.AllocationMode) (*v1.Pod, *resourceapi.ResourceClaimTemplate) { +func (b *builder) podInline() (*v1.Pod, *resourceapi.ResourceClaimTemplate) { pod := b.pod() pod.Spec.Containers[0].Name = "with-resource" podClaimName := "my-inline-claim" @@ -1517,7 +1389,6 @@ func (b *builder) podInline(allocationMode resourceapi.AllocationMode) (*v1.Pod, Kind: b.driver.claimParameterAPIKind, Name: b.parametersName(), }, - AllocationMode: allocationMode, }, }, } @@ -1525,8 +1396,8 @@ func (b *builder) podInline(allocationMode resourceapi.AllocationMode) (*v1.Pod, } // podInlineMultiple returns a pod with inline resource claim referenced by 3 containers -func (b *builder) podInlineMultiple(allocationMode resourceapi.AllocationMode) (*v1.Pod, *resourceapi.ResourceClaimTemplate) { - pod, template := b.podInline(allocationMode) +func (b *builder) podInlineMultiple() (*v1.Pod, *resourceapi.ResourceClaimTemplate) { + pod, template := b.podInline() pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy(), *pod.Spec.Containers[0].DeepCopy()) pod.Spec.Containers[1].Name = pod.Spec.Containers[1].Name + "-1" pod.Spec.Containers[2].Name = pod.Spec.Containers[1].Name + "-2" diff --git a/test/e2e/dra/test-driver/README.md b/test/e2e/dra/test-driver/README.md index baa275897c1..da67b14c589 100644 --- a/test/e2e/dra/test-driver/README.md +++ b/test/e2e/dra/test-driver/README.md @@ -78,8 +78,8 @@ configmap/pause-claim-parameters created pod/pause created $ kubectl get resourceclaims -NAME CLASSNAME ALLOCATIONMODE STATE AGE -pause-resource example WaitForFirstConsumer allocated,reserved 19s +NAME CLASSNAME STATE AGE +pause-resource example allocated,reserved 19s $ kubectl get pods NAME READY STATUS RESTARTS AGE diff --git a/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml b/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml index c6e92bb4d2e..7d2f774b55f 100644 --- a/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml +++ b/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml @@ -12,7 +12,6 @@ metadata: name: example namespace: default spec: - allocationMode: Immediate resourceClassName: example parametersRef: kind: ConfigMap diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index 360a022ec27..1661de7392c 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -409,11 +409,11 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes ExpectedEtcdPath: "/registry/resourceclasses/class1name", }, gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): { - Stub: `{"metadata": {"name": "claim1name"}, "spec": {"resourceClassName": "class1name", "allocationMode": "WaitForFirstConsumer"}}`, + Stub: `{"metadata": {"name": "claim1name"}, "spec": {"resourceClassName": "class1name"}}`, ExpectedEtcdPath: "/registry/resourceclaims/" + namespace + "/claim1name", }, gvr("resource.k8s.io", "v1alpha3", "resourceclaimtemplates"): { - Stub: `{"metadata": {"name": "claimtemplate1name"}, "spec": {"spec": {"resourceClassName": "class1name", "allocationMode": "WaitForFirstConsumer"}}}`, + Stub: `{"metadata": {"name": "claimtemplate1name"}, "spec": {"spec": {"resourceClassName": "class1name"}}}`, ExpectedEtcdPath: "/registry/resourceclaimtemplates/" + namespace + "/claimtemplate1name", }, gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): {