DRA: remove immediate allocation

As agreed in https://github.com/kubernetes/enhancements/pull/4709, immediate
allocation is one of those features which can be removed because it makes no
sense for structured parameters and the justification for classic DRA is weak.
This commit is contained in:
Patrick Ohly
2024-06-13 17:25:39 +02:00
parent b51d68bb87
commit de5742ae83
41 changed files with 395 additions and 1198 deletions

View File

@@ -15589,10 +15589,6 @@
"io.k8s.api.resource.v1alpha3.ResourceClaimSpec": {
"description": "ResourceClaimSpec defines how a resource is to be allocated.",
"properties": {
"allocationMode": {
"description": "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.",
"type": "string"
},
"parametersRef": {
"$ref": "#/definitions/io.k8s.api.resource.v1alpha3.ResourceClaimParametersReference",
"description": "ParametersRef references a separate object with arbitrary parameters that will be used by the driver when allocating a resource for the claim.\n\nThe object must be in the same namespace as the ResourceClaim."

View File

@@ -762,10 +762,6 @@
"io.k8s.api.resource.v1alpha3.ResourceClaimSpec": {
"description": "ResourceClaimSpec defines how a resource is to be allocated.",
"properties": {
"allocationMode": {
"description": "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.",
"type": "string"
},
"parametersRef": {
"allOf": [
{

View File

@@ -135,10 +135,6 @@ func TestDefaulting(t *testing.T) {
{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBindingList"}: {},
{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "RoleBinding"}: {},
{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "RoleBindingList"}: {},
{Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaim"}: {},
{Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaimList"}: {},
{Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaimTemplate"}: {},
{Group: "resource.k8s.io", Version: "v1alpha3", Kind: "ResourceClaimTemplateList"}: {},
{Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ValidatingAdmissionPolicy"}: {},
{Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ValidatingAdmissionPolicyList"}: {},
{Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ValidatingAdmissionPolicyBinding"}: {},

View File

@@ -17,24 +17,10 @@ limitations under the License.
package fuzzer
import (
fuzz "github.com/google/gofuzz"
runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/kubernetes/pkg/apis/resource"
)
// Funcs contains the fuzzer functions for the resource group.
var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
return []interface{}{
func(obj *resource.ResourceClaimSpec, c fuzz.Continue) {
c.FuzzNoCustom(obj) // fuzz self without calling this function again
// Custom fuzzing for allocation mode: pick one valid mode randomly.
modes := []resource.AllocationMode{
resource.AllocationModeImmediate,
resource.AllocationModeWaitForFirstConsumer,
}
obj.AllocationMode = modes[c.Rand.Intn(len(modes))]
},
}
return nil
}

View File

@@ -62,34 +62,8 @@ type ResourceClaimSpec struct {
// The object must be in the same namespace as the ResourceClaim.
// +optional
ParametersRef *ResourceClaimParametersReference
// Allocation can start immediately or when a Pod wants to use the
// resource. "WaitForFirstConsumer" is the default.
// +optional
AllocationMode AllocationMode
}
// AllocationMode describes whether a ResourceClaim gets allocated immediately
// when it gets created (AllocationModeImmediate) or whether allocation is
// delayed until it is needed for a Pod
// (AllocationModeWaitForFirstConsumer). Other modes might get added in the
// future.
type AllocationMode string
const (
// When a ResourceClaim has AllocationModeWaitForFirstConsumer, allocation is
// delayed until a Pod gets scheduled that needs the ResourceClaim. The
// scheduler will consider all resource requirements of that Pod and
// trigger allocation for a node that fits the Pod.
AllocationModeWaitForFirstConsumer AllocationMode = "WaitForFirstConsumer"
// When a ResourceClaim has AllocationModeImmediate, allocation starts
// as soon as the ResourceClaim gets created. This is done without
// considering the needs of Pods that will use the ResourceClaim
// because those Pods are not known yet.
AllocationModeImmediate AllocationMode = "Immediate"
)
// ResourceClaimStatus tracks whether the resource has been allocated and what
// the resulting attributes are.
type ResourceClaimStatus struct {

View File

@@ -17,16 +17,9 @@ limitations under the License.
package v1alpha3
import (
"k8s.io/api/resource/v1alpha3"
"k8s.io/apimachinery/pkg/runtime"
)
func addDefaultingFuncs(scheme *runtime.Scheme) error {
return RegisterDefaults(scheme)
}
func SetDefaults_ResourceClaimSpec(obj *v1alpha3.ResourceClaimSpec) {
if obj.AllocationMode == "" {
obj.AllocationMode = v1alpha3.AllocationModeWaitForFirstConsumer
}
}

View File

@@ -1,75 +0,0 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha3_test
import (
"reflect"
"testing"
v1alpha3 "k8s.io/api/resource/v1alpha3"
"k8s.io/apimachinery/pkg/runtime"
// ensure types are installed
"k8s.io/kubernetes/pkg/api/legacyscheme"
_ "k8s.io/kubernetes/pkg/apis/resource/install"
)
func TestSetDefaultAllocationMode(t *testing.T) {
claim := &v1alpha3.ResourceClaim{}
// field should be defaulted
defaultMode := v1alpha3.AllocationModeWaitForFirstConsumer
output := roundTrip(t, runtime.Object(claim)).(*v1alpha3.ResourceClaim)
outMode := output.Spec.AllocationMode
if outMode != defaultMode {
t.Errorf("Expected AllocationMode to be defaulted to: %+v, got: %+v", defaultMode, outMode)
}
// field should not change
nonDefaultMode := v1alpha3.AllocationModeImmediate
claim = &v1alpha3.ResourceClaim{
Spec: v1alpha3.ResourceClaimSpec{
AllocationMode: nonDefaultMode,
},
}
output = roundTrip(t, runtime.Object(claim)).(*v1alpha3.ResourceClaim)
outMode = output.Spec.AllocationMode
if outMode != v1alpha3.AllocationModeImmediate {
t.Errorf("Expected AllocationMode to remain %+v, got: %+v", nonDefaultMode, outMode)
}
}
func roundTrip(t *testing.T, obj runtime.Object) runtime.Object {
codec := legacyscheme.Codecs.LegacyCodec(v1alpha3.SchemeGroupVersion)
data, err := runtime.Encode(codec, obj)
if err != nil {
t.Errorf("%v\n %#v", err, obj)
return nil
}
obj2, err := runtime.Decode(codec, data)
if err != nil {
t.Errorf("%v\nData: %s\nSource: %#v", err, string(data), obj)
return nil
}
obj3 := reflect.New(reflect.TypeOf(obj).Elem()).Interface().(runtime.Object)
err = legacyscheme.Scheme.Convert(obj2, obj3, nil)
if err != nil {
t.Errorf("%v\nSource: %#v", err, obj2)
return nil
}
return obj3
}

View File

@@ -1159,7 +1159,6 @@ func Convert_resource_ResourceClaimSchedulingStatus_To_v1alpha3_ResourceClaimSch
func autoConvert_v1alpha3_ResourceClaimSpec_To_resource_ResourceClaimSpec(in *v1alpha3.ResourceClaimSpec, out *resource.ResourceClaimSpec, s conversion.Scope) error {
out.ResourceClassName = in.ResourceClassName
out.ParametersRef = (*resource.ResourceClaimParametersReference)(unsafe.Pointer(in.ParametersRef))
out.AllocationMode = resource.AllocationMode(in.AllocationMode)
return nil
}
@@ -1171,7 +1170,6 @@ func Convert_v1alpha3_ResourceClaimSpec_To_resource_ResourceClaimSpec(in *v1alph
func autoConvert_resource_ResourceClaimSpec_To_v1alpha3_ResourceClaimSpec(in *resource.ResourceClaimSpec, out *v1alpha3.ResourceClaimSpec, s conversion.Scope) error {
out.ResourceClassName = in.ResourceClassName
out.ParametersRef = (*v1alpha3.ResourceClaimParametersReference)(unsafe.Pointer(in.ParametersRef))
out.AllocationMode = v1alpha3.AllocationMode(in.AllocationMode)
return nil
}

View File

@@ -22,7 +22,6 @@ limitations under the License.
package v1alpha3
import (
v1alpha3 "k8s.io/api/resource/v1alpha3"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@@ -30,33 +29,5 @@ import (
// Public to allow building arbitrary schemes.
// All generated defaulters are covering - they call all nested defaulters.
func RegisterDefaults(scheme *runtime.Scheme) error {
scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaim{}, func(obj interface{}) { SetObjectDefaults_ResourceClaim(obj.(*v1alpha3.ResourceClaim)) })
scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaimList{}, func(obj interface{}) { SetObjectDefaults_ResourceClaimList(obj.(*v1alpha3.ResourceClaimList)) })
scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaimTemplate{}, func(obj interface{}) { SetObjectDefaults_ResourceClaimTemplate(obj.(*v1alpha3.ResourceClaimTemplate)) })
scheme.AddTypeDefaultingFunc(&v1alpha3.ResourceClaimTemplateList{}, func(obj interface{}) {
SetObjectDefaults_ResourceClaimTemplateList(obj.(*v1alpha3.ResourceClaimTemplateList))
})
return nil
}
func SetObjectDefaults_ResourceClaim(in *v1alpha3.ResourceClaim) {
SetDefaults_ResourceClaimSpec(&in.Spec)
}
func SetObjectDefaults_ResourceClaimList(in *v1alpha3.ResourceClaimList) {
for i := range in.Items {
a := &in.Items[i]
SetObjectDefaults_ResourceClaim(a)
}
}
func SetObjectDefaults_ResourceClaimTemplate(in *v1alpha3.ResourceClaimTemplate) {
SetDefaults_ResourceClaimSpec(&in.Spec.Spec)
}
func SetObjectDefaults_ResourceClaimTemplateList(in *v1alpha3.ResourceClaimTemplateList) {
for i := range in.Items {
a := &in.Items[i]
SetObjectDefaults_ResourceClaimTemplate(a)
}
}

View File

@@ -46,14 +46,9 @@ func validateResourceClaimSpec(spec *resource.ResourceClaimSpec, fldPath *field.
allErrs = append(allErrs, field.Invalid(fldPath.Child("resourceClassName"), spec.ResourceClassName, msg))
}
allErrs = append(allErrs, validateResourceClaimParametersRef(spec.ParametersRef, fldPath.Child("parametersRef"))...)
if !supportedAllocationModes.Has(string(spec.AllocationMode)) {
allErrs = append(allErrs, field.NotSupported(fldPath.Child("allocationMode"), spec.AllocationMode, supportedAllocationModes.List()))
}
return allErrs
}
var supportedAllocationModes = sets.NewString(string(resource.AllocationModeImmediate), string(resource.AllocationModeWaitForFirstConsumer))
// It would have been nice to use Go generics to reuse the same validation
// function for Kind and Name in both types, but generics cannot be used to
// access common fields in structs.

View File

@@ -42,14 +42,11 @@ func testClaim(name, namespace string, spec resource.ResourceClaimSpec) *resourc
}
func TestValidateClaim(t *testing.T) {
validMode := resource.AllocationModeImmediate
invalidMode := resource.AllocationMode("invalid")
goodName := "foo"
badName := "!@#$%^"
goodNS := "ns"
goodClaimSpec := resource.ResourceClaimSpec{
ResourceClassName: goodName,
AllocationMode: validMode,
}
now := metav1.Now()
badValue := "spaces not allowed"
@@ -200,14 +197,6 @@ func TestValidateClaim(t *testing.T) {
return claim
}(),
},
"bad-mode": {
wantFailures: field.ErrorList{field.NotSupported(field.NewPath("spec", "allocationMode"), invalidMode, supportedAllocationModes.List())},
claim: func() *resource.ResourceClaim {
claim := testClaim(goodName, goodNS, goodClaimSpec)
claim.Spec.AllocationMode = invalidMode
return claim
}(),
},
"good-parameters": {
claim: func() *resource.ResourceClaim {
claim := testClaim(goodName, goodNS, goodClaimSpec)
@@ -279,7 +268,6 @@ func TestValidateClaimUpdate(t *testing.T) {
}
validClaim := testClaim("foo", "ns", resource.ResourceClaimSpec{
ResourceClassName: name,
AllocationMode: resource.AllocationModeImmediate,
ParametersRef: parameters,
})
@@ -316,18 +304,6 @@ func TestValidateClaimUpdate(t *testing.T) {
return claim
},
},
"invalid-update-mode": {
wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), func() resource.ResourceClaimSpec {
spec := validClaim.Spec.DeepCopy()
spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer
return *spec
}(), "field is immutable")},
oldClaim: validClaim,
update: func(claim *resource.ResourceClaim) *resource.ResourceClaim {
claim.Spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer
return claim
},
},
}
for name, scenario := range scenarios {
@@ -343,7 +319,6 @@ func TestValidateClaimStatusUpdate(t *testing.T) {
invalidName := "!@#$%^"
validClaim := testClaim("foo", "ns", resource.ResourceClaimSpec{
ResourceClassName: "valid",
AllocationMode: resource.AllocationModeImmediate,
})
validAllocatedClaim := validClaim.DeepCopy()

View File

@@ -40,14 +40,11 @@ func testClaimTemplate(name, namespace string, spec resource.ResourceClaimSpec)
}
func TestValidateClaimTemplate(t *testing.T) {
validMode := resource.AllocationModeImmediate
invalidMode := resource.AllocationMode("invalid")
goodName := "foo"
badName := "!@#$%^"
goodNS := "ns"
goodClaimSpec := resource.ResourceClaimSpec{
ResourceClassName: goodName,
AllocationMode: validMode,
}
now := metav1.Now()
badValue := "spaces not allowed"
@@ -198,14 +195,6 @@ func TestValidateClaimTemplate(t *testing.T) {
return template
}(),
},
"bad-mode": {
wantFailures: field.ErrorList{field.NotSupported(field.NewPath("spec", "spec", "allocationMode"), invalidMode, supportedAllocationModes.List())},
template: func() *resource.ResourceClaimTemplate {
template := testClaimTemplate(goodName, goodNS, goodClaimSpec)
template.Spec.Spec.AllocationMode = invalidMode
return template
}(),
},
"good-parameters": {
template: func() *resource.ResourceClaimTemplate {
template := testClaimTemplate(goodName, goodNS, goodClaimSpec)
@@ -277,7 +266,6 @@ func TestValidateClaimTemplateUpdate(t *testing.T) {
}
validClaimTemplate := testClaimTemplate("foo", "ns", resource.ResourceClaimSpec{
ResourceClassName: name,
AllocationMode: resource.AllocationModeImmediate,
ParametersRef: parameters,
})
@@ -314,18 +302,6 @@ func TestValidateClaimTemplateUpdate(t *testing.T) {
return template
},
},
"invalid-update-mode": {
wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), func() resource.ResourceClaimTemplateSpec {
spec := validClaimTemplate.Spec.DeepCopy()
spec.Spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer
return *spec
}(), "field is immutable")},
oldClaimTemplate: validClaimTemplate,
update: func(template *resource.ResourceClaimTemplate) *resource.ResourceClaimTemplate {
template.Spec.Spec.AllocationMode = resource.AllocationModeWaitForFirstConsumer
return template
},
},
}
for name, scenario := range scenarios {

View File

@@ -329,8 +329,7 @@ func (ec *Controller) podNeedsWork(pod *v1.Pod) (bool, string) {
// - a user created a pod with spec.nodeName set, perhaps for testing
// - some scheduler was used which is unaware of DRA
// - DRA was not enabled in kube-scheduler (version skew, configuration)
if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer &&
claim.Status.Allocation == nil {
if claim.Status.Allocation == nil {
scheduling, err := ec.podSchedulingLister.PodSchedulingContexts(pod.Namespace).Get(pod.Name)
if apierrors.IsNotFound(err) {
return true, "need to create PodSchedulingContext for scheduled pod"
@@ -533,8 +532,7 @@ func (ec *Controller) syncPod(ctx context.Context, namespace, name string) error
return err
}
}
if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer &&
claim.Status.Allocation == nil {
if claim.Status.Allocation == nil {
logger.V(5).Info("create PodSchedulingContext because claim needs to be allocated", "resourceClaim", klog.KObj(claim))
return ec.ensurePodSchedulingContext(ctx, pod)
}
@@ -864,19 +862,14 @@ func (ec *Controller) syncClaim(ctx context.Context, namespace, name string) err
// for such claims and not checking for them keeps this code simpler.
if len(valid) == 0 {
if builtinControllerFinalizer >= 0 {
if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer ||
claim.DeletionTimestamp != nil {
// Allocated by scheduler with structured parameters. We can "deallocate"
// by clearing the allocation.
claim.Status.Allocation = nil
}
} else if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer {
// Allocated by scheduler with structured parameters. We can "deallocate"
// by clearing the allocation.
claim.Status.Allocation = nil
} else {
// DRA driver controller in the control plane
// needs to do the deallocation.
claim.Status.DeallocationRequested = true
}
// In all other cases, we keep the claim allocated, in particular for immediate allocation
// with a control plane controller.
}
claim, err := ec.kubeClient.ResourceV1alpha3().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})

View File

@@ -281,7 +281,7 @@ func TestSyncHandler(t *testing.T) {
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "clear-reserved-delayed-allocation",
name: "clear-reserved",
pods: []*v1.Pod{},
key: claimKey(testClaimReserved),
claims: []*resourceapi.ResourceClaim{testClaimReserved},
@@ -293,7 +293,7 @@ func TestSyncHandler(t *testing.T) {
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "clear-reserved-delayed-allocation-structured",
name: "clear-reserved-structured",
pods: []*v1.Pod{},
key: claimKey(testClaimReserved),
claims: []*resourceapi.ResourceClaim{structuredParameters(testClaimReserved)},
@@ -306,7 +306,7 @@ func TestSyncHandler(t *testing.T) {
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "dont-clear-reserved-delayed-allocation-structured",
name: "dont-clear-reserved-structured",
pods: []*v1.Pod{testPodWithResource},
key: claimKey(testClaimReserved),
claims: func() []*resourceapi.ResourceClaim {
@@ -318,50 +318,16 @@ func TestSyncHandler(t *testing.T) {
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "clear-reserved-immediate-allocation",
pods: []*v1.Pod{},
key: claimKey(testClaimReserved),
claims: func() []*resourceapi.ResourceClaim {
claim := testClaimReserved.DeepCopy()
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
return []*resourceapi.ResourceClaim{claim}
}(),
expectedClaims: func() []resourceapi.ResourceClaim {
claim := testClaimAllocated.DeepCopy()
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
return []resourceapi.ResourceClaim{*claim}
}(),
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "clear-reserved-immediate-allocation-structured",
name: "clear-reserved-structured-deleted",
pods: []*v1.Pod{},
key: claimKey(testClaimReserved),
claims: func() []*resourceapi.ResourceClaim {
claim := structuredParameters(testClaimReserved.DeepCopy())
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
return []*resourceapi.ResourceClaim{claim}
}(),
expectedClaims: func() []resourceapi.ResourceClaim {
claim := structuredParameters(testClaimAllocated.DeepCopy())
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
return []resourceapi.ResourceClaim{*claim}
}(),
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "clear-reserved-immediate-allocation-structured-deleted",
pods: []*v1.Pod{},
key: claimKey(testClaimReserved),
claims: func() []*resourceapi.ResourceClaim {
claim := structuredParameters(testClaimReserved.DeepCopy())
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
claim.DeletionTimestamp = &metav1.Time{}
return []*resourceapi.ResourceClaim{claim}
}(),
expectedClaims: func() []resourceapi.ResourceClaim {
claim := structuredParameters(testClaimAllocated.DeepCopy())
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
claim.DeletionTimestamp = &metav1.Time{}
claim.Finalizers = []string{}
claim.Status.Allocation = nil
@@ -370,18 +336,16 @@ func TestSyncHandler(t *testing.T) {
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "immediate-allocation-structured-deleted",
name: "structured-deleted",
pods: []*v1.Pod{},
key: claimKey(testClaimReserved),
claims: func() []*resourceapi.ResourceClaim {
claim := structuredParameters(testClaimAllocated.DeepCopy())
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
claim.DeletionTimestamp = &metav1.Time{}
return []*resourceapi.ResourceClaim{claim}
}(),
expectedClaims: func() []resourceapi.ResourceClaim {
claim := structuredParameters(testClaimAllocated.DeepCopy())
claim.Spec.AllocationMode = resourceapi.AllocationModeImmediate
claim.DeletionTimestamp = &metav1.Time{}
claim.Finalizers = []string{}
claim.Status.Allocation = nil
@@ -390,7 +354,7 @@ func TestSyncHandler(t *testing.T) {
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "clear-reserved-when-done-delayed-allocation",
name: "clear-reserved-when-done",
pods: func() []*v1.Pod {
pods := []*v1.Pod{testPodWithResource.DeepCopy()}
pods[0].Status.Phase = v1.PodSucceeded
@@ -410,28 +374,6 @@ func TestSyncHandler(t *testing.T) {
}(),
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "clear-reserved-when-done-immediate-allocation",
pods: func() []*v1.Pod {
pods := []*v1.Pod{testPodWithResource.DeepCopy()}
pods[0].Status.Phase = v1.PodSucceeded
return pods
}(),
key: claimKey(testClaimReserved),
claims: func() []*resourceapi.ResourceClaim {
claims := []*resourceapi.ResourceClaim{testClaimReserved.DeepCopy()}
claims[0].OwnerReferences = nil
claims[0].Spec.AllocationMode = resourceapi.AllocationModeImmediate
return claims
}(),
expectedClaims: func() []resourceapi.ResourceClaim {
claims := []resourceapi.ResourceClaim{*testClaimAllocated.DeepCopy()}
claims[0].OwnerReferences = nil
claims[0].Spec.AllocationMode = resourceapi.AllocationModeImmediate
return claims
}(),
expectedMetrics: expectedMetrics{0, 0},
},
{
name: "remove-reserved",
pods: []*v1.Pod{testPod},
@@ -587,7 +529,6 @@ func makeClaim(name, namespace, classname string, owner *metav1.OwnerReference)
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
Spec: resourceapi.ResourceClaimSpec{
ResourceClassName: classname,
AllocationMode: resourceapi.AllocationModeWaitForFirstConsumer,
},
}
if owner != nil {
@@ -607,7 +548,6 @@ func makeGeneratedClaim(podClaimName, generateName, namespace, classname string,
},
Spec: resourceapi.ResourceClaimSpec{
ResourceClassName: classname,
AllocationMode: resourceapi.AllocationModeWaitForFirstConsumer,
},
}
if owner != nil {
@@ -709,10 +649,6 @@ func normalizeClaims(claims []resourceapi.ResourceClaim) []resourceapi.ResourceC
if len(claims[i].Status.ReservedFor) == 0 {
claims[i].Status.ReservedFor = nil
}
if claims[i].Spec.AllocationMode == "" {
// This emulates defaulting.
claims[i].Spec.AllocationMode = resourceapi.AllocationModeWaitForFirstConsumer
}
}
return claims
}

View File

@@ -46572,13 +46572,6 @@ func schema_k8sio_api_resource_v1alpha3_ResourceClaimSpec(ref common.ReferenceCa
Ref: ref("k8s.io/api/resource/v1alpha3.ResourceClaimParametersReference"),
},
},
"allocationMode": {
SchemaProps: spec.SchemaProps{
Description: "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.",
Type: []string{"string"},
Format: "",
},
},
},
Required: []string{"resourceClassName"},
},

View File

@@ -634,7 +634,6 @@ func AddHandlers(h printers.PrintHandler) {
resourceClaimColumnDefinitions := []metav1.TableColumnDefinition{
{Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]},
{Name: "ResourceClassName", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["resourceClassName"]},
{Name: "AllocationMode", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["allocationMode"]},
{Name: "State", Type: "string", Description: "A summary of the current state (allocated, pending, reserved, etc.)."},
{Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]},
}
@@ -644,7 +643,6 @@ func AddHandlers(h printers.PrintHandler) {
resourceClaimTemplateColumnDefinitions := []metav1.TableColumnDefinition{
{Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]},
{Name: "ResourceClassName", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["resourceClassName"]},
{Name: "AllocationMode", Type: "string", Description: resourceapi.ResourceClaimSpec{}.SwaggerDoc()["allocationMode"]},
{Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]},
}
_ = h.TableHandler(resourceClaimTemplateColumnDefinitions, printResourceClaimTemplate)
@@ -3006,7 +3004,7 @@ func printResourceClaim(obj *resource.ResourceClaim, options printers.GenerateOp
row := metav1.TableRow{
Object: runtime.RawExtension{Object: obj},
}
row.Cells = append(row.Cells, obj.Name, obj.Spec.ResourceClassName, string(obj.Spec.AllocationMode), resourceClaimState(obj), translateTimestampSince(obj.CreationTimestamp))
row.Cells = append(row.Cells, obj.Name, obj.Spec.ResourceClassName, resourceClaimState(obj), translateTimestampSince(obj.CreationTimestamp))
return []metav1.TableRow{row}, nil
}
@@ -3047,7 +3045,7 @@ func printResourceClaimTemplate(obj *resource.ResourceClaimTemplate, options pri
row := metav1.TableRow{
Object: runtime.RawExtension{Object: obj},
}
row.Cells = append(row.Cells, obj.Name, obj.Spec.Spec.ResourceClassName, string(obj.Spec.Spec.AllocationMode), translateTimestampSince(obj.CreationTimestamp))
row.Cells = append(row.Cells, obj.Name, obj.Spec.Spec.ResourceClassName, translateTimestampSince(obj.CreationTimestamp))
return []metav1.TableRow{row}, nil
}

View File

@@ -58,7 +58,6 @@ func validNewClaim(name, ns string) *resource.ResourceClaim {
},
Spec: resource.ResourceClaimSpec{
ResourceClassName: "example",
AllocationMode: resource.AllocationModeImmediate,
},
Status: resource.ResourceClaimStatus{},
}

View File

@@ -31,7 +31,6 @@ var resourceClaim = &resource.ResourceClaim{
},
Spec: resource.ResourceClaimSpec{
ResourceClassName: "valid-class",
AllocationMode: resource.AllocationModeImmediate,
},
}

View File

@@ -55,7 +55,6 @@ func validNewClaimTemplate(name string) *resource.ResourceClaimTemplate {
Spec: resource.ResourceClaimTemplateSpec{
Spec: resource.ResourceClaimSpec{
ResourceClassName: "valid-class",
AllocationMode: resource.AllocationModeImmediate,
},
},
}

View File

@@ -32,7 +32,6 @@ var resourceClaimTemplate = &resource.ResourceClaimTemplate{
Spec: resource.ResourceClaimTemplateSpec{
Spec: resource.ResourceClaimSpec{
ResourceClassName: "valid-class",
AllocationMode: resource.AllocationModeImmediate,
},
},
}

View File

@@ -713,8 +713,7 @@ func (pl *dynamicResources) isSchedulableAfterPodSchedulingContextChange(logger
// we allow backoff.
pendingDelayedClaims := 0
if err := pl.foreachPodResourceClaim(pod, func(podResourceName string, claim *resourceapi.ResourceClaim) {
if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer &&
claim.Status.Allocation == nil &&
if claim.Status.Allocation == nil &&
!podSchedulingHasClaimInfo(podScheduling, podResourceName) {
pendingDelayedClaims++
}
@@ -970,9 +969,6 @@ func (pl *dynamicResources) PreFilter(ctx context.Context, state *framework.Cycl
}
s.informationsForClaim[index].controller = controller
needResourceInformation = true
} else if claim.Spec.AllocationMode == resourceapi.AllocationModeImmediate {
// This will get resolved by the resource driver.
return nil, statusUnschedulable(logger, "unallocated immediate resourceclaim", "pod", klog.KObj(pod), "resourceclaim", klog.KObj(claim))
}
}
}
@@ -1161,74 +1157,63 @@ func (pl *dynamicResources) Filter(ctx context.Context, cs *framework.CycleState
var unavailableClaims []int
for index, claim := range state.claims {
logger.V(10).Info("filtering based on resource claims of the pod", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim))
switch {
case claim.Status.Allocation != nil:
if claim.Status.Allocation != nil {
if nodeSelector := state.informationsForClaim[index].availableOnNode; nodeSelector != nil {
if !nodeSelector.Match(node) {
logger.V(5).Info("AvailableOnNodes does not match", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim))
unavailableClaims = append(unavailableClaims, index)
}
}
case claim.Status.DeallocationRequested:
continue
}
if claim.Status.DeallocationRequested {
// We shouldn't get here. PreFilter already checked this.
return statusUnschedulable(logger, "resourceclaim must be reallocated", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim))
case claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer ||
state.informationsForClaim[index].structuredParameters:
if selector := state.informationsForClaim[index].availableOnNode; selector != nil {
if matches := selector.Match(node); !matches {
return statusUnschedulable(logger, "excluded by resource class node filter", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclassName", claim.Spec.ResourceClassName)
}
}
if selector := state.informationsForClaim[index].availableOnNode; selector != nil {
if matches := selector.Match(node); !matches {
return statusUnschedulable(logger, "excluded by resource class node filter", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclassName", claim.Spec.ResourceClassName)
}
// Can the builtin controller tell us whether the node is suitable?
if state.informationsForClaim[index].structuredParameters {
suitable, err := state.informationsForClaim[index].controller.nodeIsSuitable(ctx, node.Name, state.resources)
if err != nil {
// An error indicates that something wasn't configured correctly, for example
// writing a CEL expression which doesn't handle a map lookup error. Normally
// this should never fail. We could return an error here, but then the pod
// would get retried. Instead we ignore the node.
return statusUnschedulable(logger, fmt.Sprintf("checking structured parameters failed: %v", err), "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim))
}
if !suitable {
return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim))
}
} else {
if status := state.informationsForClaim[index].status; status != nil {
for _, unsuitableNode := range status.UnsuitableNodes {
if node.Name == unsuitableNode {
return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim), "unsuitablenodes", status.UnsuitableNodes)
}
}
// Can the builtin controller tell us whether the node is suitable?
if state.informationsForClaim[index].structuredParameters {
suitable, err := state.informationsForClaim[index].controller.nodeIsSuitable(ctx, node.Name, state.resources)
if err != nil {
// An error indicates that something wasn't configured correctly, for example
// writing a CEL expression which doesn't handle a map lookup error. Normally
// this should never fail. We could return an error here, but then the pod
// would get retried. Instead we ignore the node.
return statusUnschedulable(logger, fmt.Sprintf("checking structured parameters failed: %v", err), "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim))
}
if !suitable {
return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim))
}
} else {
if status := state.informationsForClaim[index].status; status != nil {
for _, unsuitableNode := range status.UnsuitableNodes {
if node.Name == unsuitableNode {
return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim), "unsuitablenodes", status.UnsuitableNodes)
}
}
}
default:
// This claim should have been handled above.
// Immediate allocation with control plane controller
// was already checked for in PreFilter.
return statusError(logger, fmt.Errorf("internal error, unexpected allocation mode %v", claim.Spec.AllocationMode))
}
}
if len(unavailableClaims) > 0 {
// Remember all unavailable claims. This might be observed
// concurrently, so we have to lock the state before writing.
state.mutex.Lock()
defer state.mutex.Unlock()
if state.unavailableClaims == nil {
state.unavailableClaims = sets.New[int]()
}
for _, index := range unavailableClaims {
claim := state.claims[index]
// Deallocation makes more sense for claims with
// delayed allocation. Claims with immediate allocation
// would just get allocated again for a random node,
// which is unlikely to help the pod.
//
// Claims with builtin controller are handled like
// claims with delayed allocation.
if claim.Spec.AllocationMode == resourceapi.AllocationModeWaitForFirstConsumer ||
state.informationsForClaim[index].controller != nil {
state.unavailableClaims.Insert(index)
}
state.unavailableClaims.Insert(index)
}
return statusUnschedulable(logger, "resourceclaim not available on the node", "pod", klog.KObj(pod))
}

View File

@@ -165,53 +165,39 @@ var (
Namespace(namespace).
ResourceClassName(className).
Obj()
pendingImmediateClaim = st.FromResourceClaim(claim).
AllocationMode(resourceapi.AllocationModeImmediate).
Obj()
structuredAllocatedImmediateClaim = st.FromResourceClaim(pendingImmediateClaim).
Allocation("some-driver", &resourceapi.AllocationResult{}).
Structured("worker", "instance-1").
Obj()
pendingDelayedClaim = st.FromResourceClaim(claim).
OwnerReference(podName, podUID, podKind).
AllocationMode(resourceapi.AllocationModeWaitForFirstConsumer).
Obj()
pendingDelayedClaim2 = st.FromResourceClaim(pendingDelayedClaim).
Name(claimName2).
Obj()
deallocatingClaim = st.FromResourceClaim(pendingImmediateClaim).
pendingClaim = st.FromResourceClaim(claim).
OwnerReference(podName, podUID, podKind).
Obj()
pendingClaim2 = st.FromResourceClaim(pendingClaim).
Name(claimName2).
Obj()
deallocatingClaim = st.FromResourceClaim(pendingClaim).
Allocation("some-driver", &resourceapi.AllocationResult{}).
DeallocationRequested(true).
Obj()
inUseClaim = st.FromResourceClaim(pendingImmediateClaim).
inUseClaim = st.FromResourceClaim(pendingClaim).
Allocation("some-driver", &resourceapi.AllocationResult{}).
ReservedForPod(podName, types.UID(podUID)).
Obj()
structuredInUseClaim = st.FromResourceClaim(inUseClaim).
Structured("worker", "instance-1").
Obj()
allocatedClaim = st.FromResourceClaim(pendingDelayedClaim).
allocatedClaim = st.FromResourceClaim(pendingClaim).
Allocation("some-driver", &resourceapi.AllocationResult{}).
Obj()
pendingDelayedClaimWithParams = st.FromResourceClaim(pendingDelayedClaim).ParametersRef(claimName).Obj()
pendingClaimWithParams = st.FromResourceClaim(pendingClaim).ParametersRef(claimName).Obj()
structuredAllocatedClaim = st.FromResourceClaim(allocatedClaim).Structured("worker", "instance-1").Obj()
structuredAllocatedClaimWithParams = st.FromResourceClaim(structuredAllocatedClaim).ParametersRef(claimName).Obj()
otherStructuredAllocatedClaim = st.FromResourceClaim(structuredAllocatedClaim).Name(structuredAllocatedClaim.Name + "-other").Obj()
allocatedDelayedClaimWithWrongTopology = st.FromResourceClaim(allocatedClaim).
Allocation("some-driver", &resourceapi.AllocationResult{AvailableOnNodes: st.MakeNodeSelector().In("no-such-label", []string{"no-such-value"}).Obj()}).
Obj()
structuredAllocatedDelayedClaimWithWrongTopology = st.FromResourceClaim(allocatedDelayedClaimWithWrongTopology).
Structured("worker-2", "instance-1").
Obj()
allocatedImmediateClaimWithWrongTopology = st.FromResourceClaim(allocatedDelayedClaimWithWrongTopology).
AllocationMode(resourceapi.AllocationModeImmediate).
allocatedClaimWithWrongTopology = st.FromResourceClaim(allocatedClaim).
Allocation("some-driver", &resourceapi.AllocationResult{AvailableOnNodes: st.MakeNodeSelector().In("no-such-label", []string{"no-such-value"}).Obj()}).
Obj()
structuredAllocatedClaimWithWrongTopology = st.FromResourceClaim(allocatedClaimWithWrongTopology).
Structured("worker-2", "instance-1").
Obj()
structuredAllocatedImmediateClaimWithWrongTopology = st.FromResourceClaim(allocatedImmediateClaimWithWrongTopology).
Structured("worker-2", "instance-1").
Obj()
allocatedClaimWithGoodTopology = st.FromResourceClaim(allocatedClaim).
Allocation("some-driver", &resourceapi.AllocationResult{AvailableOnNodes: st.MakeNodeSelector().In("kubernetes.io/hostname", []string{"worker"}).Obj()}).
Obj()
@@ -483,22 +469,9 @@ func TestPlugin(t *testing.T) {
},
},
},
"waiting-for-immediate-allocation": {
"structured-no-resources": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingImmediateClaim},
classes: []*resourceapi.ResourceClass{resourceClass},
want: want{
prefilter: result{
status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `unallocated immediate resourceclaim`),
},
postfilter: result{
status: framework.NewStatus(framework.Unschedulable, `no new claims to deallocate`),
},
},
},
"immediate-allocation-structured-no-resources": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingImmediateClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
classes: []*resourceapi.ResourceClass{structuredResourceClass},
want: want{
filter: perNodeResult{
@@ -511,51 +484,9 @@ func TestPlugin(t *testing.T) {
},
},
},
"immediate-allocation-structured-with-resources": {
"structured-with-resources": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingImmediateClaim},
classes: []*resourceapi.ResourceClass{structuredResourceClass},
objs: []apiruntime.Object{workerNodeSlice},
want: want{
reserve: result{
inFlightClaim: structuredAllocatedImmediateClaim,
},
prebind: result{
assumedClaim: reserve(structuredAllocatedImmediateClaim, podWithClaimName),
changes: change{
claim: func(claim *resourceapi.ResourceClaim) *resourceapi.ResourceClaim {
if claim.Name == claimName {
claim = claim.DeepCopy()
claim.Finalizers = structuredAllocatedImmediateClaim.Finalizers
claim.Status = structuredInUseClaim.Status
}
return claim
},
},
},
postbind: result{
assumedClaim: reserve(structuredAllocatedImmediateClaim, podWithClaimName),
},
},
},
"delayed-allocation-structured-no-resources": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
classes: []*resourceapi.ResourceClass{structuredResourceClass},
want: want{
filter: perNodeResult{
workerNode.Name: {
status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `resourceclaim cannot be allocated for the node (unsuitable)`),
},
},
postfilter: result{
status: framework.NewStatus(framework.Unschedulable, `still not schedulable`),
},
},
},
"delayed-allocation-structured-with-resources": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
classes: []*resourceapi.ResourceClass{structuredResourceClass},
objs: []apiruntime.Object{workerNodeSlice},
want: want{
@@ -580,12 +511,12 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-structured-with-resources-has-finalizer": {
"structured-with-resources-has-finalizer": {
// As before. but the finalizer is already set. Could happen if
// the scheduler got interrupted.
pod: podWithClaimName,
claims: func() []*resourceapi.ResourceClaim {
claim := pendingDelayedClaim.DeepCopy()
claim := pendingClaim.DeepCopy()
claim.Finalizers = structuredAllocatedClaim.Finalizers
return []*resourceapi.ResourceClaim{claim}
}(),
@@ -612,12 +543,12 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-structured-with-resources-finalizer-gets-removed": {
"structured-with-resources-finalizer-gets-removed": {
// As before. but the finalizer is already set. Then it gets
// removed before the scheduler reaches PreBind.
pod: podWithClaimName,
claims: func() []*resourceapi.ResourceClaim {
claim := pendingDelayedClaim.DeepCopy()
claim := pendingClaim.DeepCopy()
claim.Finalizers = structuredAllocatedClaim.Finalizers
return []*resourceapi.ResourceClaim{claim}
}(),
@@ -653,11 +584,11 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-structured-with-resources-finalizer-gets-added": {
"structured-with-resources-finalizer-gets-added": {
// No finalizer initially, then it gets added before
// the scheduler reaches PreBind. Shouldn't happen?
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
classes: []*resourceapi.ResourceClass{structuredResourceClass},
objs: []apiruntime.Object{workerNodeSlice},
prepare: prepare{
@@ -689,9 +620,9 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-structured-skip-bind": {
"structured-skip-bind": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
classes: []*resourceapi.ResourceClass{structuredResourceClass},
objs: []apiruntime.Object{workerNodeSlice},
want: want{
@@ -701,9 +632,9 @@ func TestPlugin(t *testing.T) {
unreserveBeforePreBind: &result{},
},
},
"delayed-allocation-structured-exhausted-resources": {
"structured-exhausted-resources": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim, otherStructuredAllocatedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim, otherStructuredAllocatedClaim},
classes: []*resourceapi.ResourceClass{structuredResourceClass},
objs: []apiruntime.Object{workerNodeSlice},
want: want{
@@ -720,7 +651,7 @@ func TestPlugin(t *testing.T) {
"with-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams},
claims: []*resourceapi.ResourceClaim{pendingClaimWithParams},
classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams},
objs: []apiruntime.Object{claimParameters, classParameters, workerNodeSlice},
want: want{
@@ -748,7 +679,7 @@ func TestPlugin(t *testing.T) {
"with-translated-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)},
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)},
classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)},
objs: []apiruntime.Object{claimParameters, claimParametersOtherNamespace /* must be ignored */, classParameters, workerNodeSlice},
want: want{
@@ -776,7 +707,7 @@ func TestPlugin(t *testing.T) {
"missing-class-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams},
claims: []*resourceapi.ResourceClaim{pendingClaimWithParams},
classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams},
objs: []apiruntime.Object{claimParameters, workerNodeSlice},
want: want{
@@ -791,7 +722,7 @@ func TestPlugin(t *testing.T) {
"missing-claim-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams},
claims: []*resourceapi.ResourceClaim{pendingClaimWithParams},
classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams},
objs: []apiruntime.Object{classParameters, workerNodeSlice},
want: want{
@@ -806,7 +737,7 @@ func TestPlugin(t *testing.T) {
"missing-translated-class-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)},
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)},
classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)},
objs: []apiruntime.Object{claimParameters, workerNodeSlice},
want: want{
@@ -821,7 +752,7 @@ func TestPlugin(t *testing.T) {
"missing-translated-claim-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)},
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)},
classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)},
objs: []apiruntime.Object{classParameters, workerNodeSlice},
want: want{
@@ -836,7 +767,7 @@ func TestPlugin(t *testing.T) {
"too-many-translated-class-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)},
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)},
classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)},
objs: []apiruntime.Object{claimParameters, classParameters, st.FromClassParameters(classParameters).Name("other").Obj() /* too many */, workerNodeSlice},
want: want{
@@ -851,7 +782,7 @@ func TestPlugin(t *testing.T) {
"too-many-translated-claim-parameters": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingDelayedClaimWithParams)},
claims: []*resourceapi.ResourceClaim{claimWithCRD(pendingClaimWithParams)},
classes: []*resourceapi.ResourceClass{classWithCRD(structuredResourceClassWithCRD)},
objs: []apiruntime.Object{claimParameters, st.FromClaimParameters(claimParameters).Name("other").Obj() /* too many */, classParameters, workerNodeSlice},
want: want{
@@ -866,7 +797,7 @@ func TestPlugin(t *testing.T) {
"claim-parameters-CEL-runtime-error": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams},
claims: []*resourceapi.ResourceClaim{pendingClaimWithParams},
classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams},
objs: []apiruntime.Object{breakCELInClaimParameters(claimParameters), classParameters, workerNodeSlice},
want: want{
@@ -883,7 +814,7 @@ func TestPlugin(t *testing.T) {
"class-parameters-CEL-runtime-error": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaimWithParams},
claims: []*resourceapi.ResourceClaim{pendingClaimWithParams},
classes: []*resourceapi.ResourceClass{structuredResourceClassWithParams},
objs: []apiruntime.Object{claimParameters, breakCELInClassParameters(classParameters), workerNodeSlice},
want: want{
@@ -910,9 +841,9 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-missing-class": {
"missing-class": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
want: want{
prefilter: result{
status: framework.NewStatus(framework.UnschedulableAndUnresolvable, fmt.Sprintf("resource class %s does not exist", className)),
@@ -922,11 +853,11 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-scheduling-select-immediately": {
"scheduling-select-immediately": {
// Create the PodSchedulingContext object, ask for information
// and select a node.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
classes: []*resourceapi.ResourceClass{resourceClass},
want: want{
prebind: result{
@@ -935,12 +866,12 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-scheduling-ask": {
"scheduling-ask": {
// Create the PodSchedulingContext object, ask for
// information, but do not select a node because
// there are multiple claims.
pod: podWithTwoClaimNames,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim, pendingDelayedClaim2},
claims: []*resourceapi.ResourceClaim{pendingClaim, pendingClaim2},
classes: []*resourceapi.ResourceClass{resourceClass},
want: want{
prebind: result{
@@ -949,11 +880,11 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-scheduling-finish": {
"scheduling-finish": {
// Use the populated PodSchedulingContext object to select a
// node.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
schedulings: []*resourceapi.PodSchedulingContext{schedulingInfo},
classes: []*resourceapi.ResourceClass{resourceClass},
want: want{
@@ -969,11 +900,11 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-scheduling-finish-concurrent-label-update": {
"scheduling-finish-concurrent-label-update": {
// Use the populated PodSchedulingContext object to select a
// node.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
schedulings: []*resourceapi.PodSchedulingContext{schedulingInfo},
classes: []*resourceapi.ResourceClass{resourceClass},
prepare: prepare{
@@ -994,7 +925,7 @@ func TestPlugin(t *testing.T) {
},
},
},
"delayed-allocation-scheduling-completed": {
"scheduling-completed": {
// Remove PodSchedulingContext object once the pod is scheduled.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{allocatedClaim},
@@ -1031,11 +962,11 @@ func TestPlugin(t *testing.T) {
},
},
},
"wrong-topology-delayed-allocation": {
"wrong-topology": {
// PostFilter tries to get the pod scheduleable by
// deallocating the claim.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{allocatedDelayedClaimWithWrongTopology},
claims: []*resourceapi.ResourceClaim{allocatedClaimWithWrongTopology},
want: want{
filter: perNodeResult{
workerNode.Name: {
@@ -1055,29 +986,11 @@ func TestPlugin(t *testing.T) {
},
},
},
"wrong-topology-immediate-allocation": {
"wrong-topology-structured": {
// PostFilter tries to get the pod scheduleable by
// deallocating the claim.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{allocatedImmediateClaimWithWrongTopology},
want: want{
filter: perNodeResult{
workerNode.Name: {
status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `resourceclaim not available on the node`),
},
},
postfilter: result{
// Claims with immediate allocation don't. They would just get allocated again right
// away, without considering the needs of the pod.
status: framework.NewStatus(framework.Unschedulable, `still not schedulable`),
},
},
},
"wrong-topology-delayed-allocation-structured": {
// PostFilter tries to get the pod scheduleable by
// deallocating the claim.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{structuredAllocatedDelayedClaimWithWrongTopology},
claims: []*resourceapi.ResourceClaim{structuredAllocatedClaimWithWrongTopology},
want: want{
filter: perNodeResult{
workerNode.Name: {
@@ -1097,25 +1010,6 @@ func TestPlugin(t *testing.T) {
},
},
},
"wrong-topology-immediate-allocation-structured": {
// PostFilter tries to get the pod scheduleable by
// deallocating the claim.
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{structuredAllocatedImmediateClaimWithWrongTopology},
want: want{
filter: perNodeResult{
workerNode.Name: {
status: framework.NewStatus(framework.UnschedulableAndUnresolvable, `resourceclaim not available on the node`),
},
},
postfilter: result{
// Claims with immediate allocation don't. The allocation is considered
// more important than the pod and pods need to wait for the node to
// become available again.
status: framework.NewStatus(framework.Unschedulable, `still not schedulable`),
},
},
},
"good-topology": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{allocatedClaimWithGoodTopology},
@@ -1665,22 +1559,22 @@ func Test_isSchedulableAfterClaimChange(t *testing.T) {
},
"queue-on-add": {
pod: podWithClaimName,
newObj: pendingImmediateClaim,
newObj: pendingClaim,
expectedHint: framework.Queue,
},
"backoff-wrong-old-object": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: "not-a-claim",
newObj: pendingImmediateClaim,
newObj: pendingClaim,
expectedErr: true,
},
"skip-adding-finalizer": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingImmediateClaim},
oldObj: pendingImmediateClaim,
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: pendingClaim,
newObj: func() *resourceapi.ResourceClaim {
claim := pendingImmediateClaim.DeepCopy()
claim := pendingClaim.DeepCopy()
claim.Finalizers = append(claim.Finalizers, "foo")
return claim
}(),
@@ -1688,10 +1582,10 @@ func Test_isSchedulableAfterClaimChange(t *testing.T) {
},
"queue-on-status-change": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingImmediateClaim},
oldObj: pendingImmediateClaim,
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: pendingClaim,
newObj: func() *resourceapi.ResourceClaim {
claim := pendingImmediateClaim.DeepCopy()
claim := pendingClaim.DeepCopy()
claim.Status.Allocation = &resourceapi.AllocationResult{}
return claim
}(),
@@ -1699,7 +1593,7 @@ func Test_isSchedulableAfterClaimChange(t *testing.T) {
},
"structured-claim-deallocate": {
pod: podWithClaimName,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim, otherStructuredAllocatedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim, otherStructuredAllocatedClaim},
oldObj: otherStructuredAllocatedClaim,
newObj: func() *resourceapi.ResourceClaim {
claim := otherStructuredAllocatedClaim.DeepCopy()
@@ -1804,7 +1698,7 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) {
},
"skip-unrelated-object": {
pod: podWithClaimTemplate,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
newObj: func() *resourceapi.PodSchedulingContext {
scheduling := scheduling.DeepCopy()
scheduling.Name += "-foo"
@@ -1826,21 +1720,21 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) {
},
"skip-missing-infos": {
pod: podWithClaimTemplateInStatus,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: scheduling,
newObj: scheduling,
expectedHint: framework.QueueSkip,
},
"queue-new-infos": {
pod: podWithClaimTemplateInStatus,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: scheduling,
newObj: schedulingInfo,
expectedHint: framework.Queue,
},
"queue-bad-selected-node": {
pod: podWithClaimTemplateInStatus,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: func() *resourceapi.PodSchedulingContext {
scheduling := schedulingInfo.DeepCopy()
scheduling.Spec.SelectedNode = workerNode.Name
@@ -1856,7 +1750,7 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) {
},
"skip-spec-changes": {
pod: podWithClaimTemplateInStatus,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: schedulingInfo,
newObj: func() *resourceapi.PodSchedulingContext {
scheduling := schedulingInfo.DeepCopy()
@@ -1867,7 +1761,7 @@ func Test_isSchedulableAfterPodSchedulingContextChange(t *testing.T) {
},
"backoff-other-changes": {
pod: podWithClaimTemplateInStatus,
claims: []*resourceapi.ResourceClaim{pendingDelayedClaim},
claims: []*resourceapi.ResourceClaim{pendingClaim},
oldObj: schedulingInfo,
newObj: func() *resourceapi.PodSchedulingContext {
scheduling := schedulingInfo.DeepCopy()

View File

@@ -946,12 +946,6 @@ func (wrapper *ResourceClaimWrapper) OwnerReference(name, uid string, gvk schema
return wrapper
}
// AllocationMode sets the allocation mode of the inner object.
func (wrapper *ResourceClaimWrapper) AllocationMode(a resourceapi.AllocationMode) *ResourceClaimWrapper {
wrapper.ResourceClaim.Spec.AllocationMode = a
return wrapper
}
// ParametersRef sets a reference to a ResourceClaimParameters.resource.k8s.io.
func (wrapper *ResourceClaimWrapper) ParametersRef(name string) *ResourceClaimWrapper {
wrapper.ResourceClaim.Spec.ParametersRef = &resourceapi.ResourceClaimParametersReference{

View File

@@ -1331,148 +1331,146 @@ func init() {
}
var fileDescriptor_66649ee9bbcd89d2 = []byte{
// 2242 bytes of a gzipped FileDescriptorProto
// 2219 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x1a, 0x4d, 0x6c, 0x1c, 0x57,
0xd9, 0xb3, 0xbb, 0x89, 0xd7, 0x9f, 0xed, 0xb5, 0x33, 0xb6, 0xe3, 0x4d, 0xea, 0xee, 0x6e, 0x47,
0x20, 0x2c, 0x70, 0x76, 0x1b, 0xa7, 0x4d, 0xa3, 0x52, 0x90, 0x32, 0x71, 0x13, 0x2c, 0x9a, 0xd4,
0x7d, 0x4b, 0xdc, 0xa6, 0xfc, 0x75, 0xbc, 0xf3, 0x62, 0x0f, 0xd9, 0x9d, 0xd9, 0xcc, 0x7b, 0xeb,
0x26, 0xe2, 0x12, 0x55, 0x20, 0xb8, 0x20, 0x15, 0x81, 0x10, 0x9c, 0x38, 0x21, 0xc4, 0x85, 0x0b,
0x5c, 0x39, 0x55, 0xd0, 0x1c, 0x83, 0x40, 0xa2, 0xe2, 0xb0, 0x22, 0xcb, 0x91, 0x23, 0xb7, 0x9e,
0xd0, 0xbc, 0xf7, 0xe6, 0xe7, 0xcd, 0xce, 0xac, 0x77, 0x96, 0xc6, 0x4a, 0x4e, 0xde, 0x79, 0xef,
0xfb, 0x7b, 0xdf, 0xff, 0x7b, 0x9f, 0x61, 0xe3, 0xce, 0x25, 0x52, 0xb7, 0x9c, 0x86, 0xd1, 0xb5,
0x1a, 0x2e, 0x26, 0x4e, 0xcf, 0x6d, 0xe1, 0xc6, 0xe1, 0x79, 0xa3, 0xdd, 0x3d, 0x30, 0x2e, 0x34,
0xf6, 0xb1, 0x8d, 0x5d, 0x83, 0x62, 0xb3, 0xde, 0x75, 0x1d, 0xea, 0xa8, 0x6b, 0x1c, 0xba, 0x6e,
0x74, 0xad, 0xba, 0x0f, 0x5d, 0xf7, 0xa1, 0xcf, 0x9e, 0xdb, 0xb7, 0xe8, 0x41, 0x6f, 0xaf, 0xde,
0x72, 0x3a, 0x8d, 0x7d, 0x67, 0xdf, 0x69, 0x30, 0xa4, 0xbd, 0xde, 0x6d, 0xf6, 0xc5, 0x3e, 0xd8,
0x2f, 0x4e, 0xec, 0xac, 0x16, 0x61, 0xdd, 0x72, 0x5c, 0x8f, 0x6d, 0x9c, 0xe1, 0xd9, 0x97, 0x42,
0x98, 0x8e, 0xd1, 0x3a, 0xb0, 0x6c, 0xec, 0xde, 0x6f, 0x74, 0xef, 0xec, 0xcb, 0xf2, 0x66, 0xc1,
0x22, 0x8d, 0x0e, 0xa6, 0x46, 0x12, 0xaf, 0x46, 0x1a, 0x96, 0xdb, 0xb3, 0xa9, 0xd5, 0x19, 0x66,
0x73, 0xf1, 0x28, 0x04, 0xd2, 0x3a, 0xc0, 0x1d, 0x23, 0x8e, 0xa7, 0xfd, 0x32, 0x07, 0x8b, 0x97,
0xdb, 0x6d, 0xa7, 0x65, 0x50, 0xcb, 0xb1, 0x11, 0x26, 0xbd, 0x36, 0x55, 0x1d, 0x58, 0xf0, 0xcf,
0xf3, 0x35, 0xc3, 0x36, 0xdb, 0x98, 0x94, 0x95, 0x5a, 0x7e, 0x7d, 0x76, 0x73, 0xa3, 0x3e, 0x4a,
0xe9, 0x75, 0x24, 0x21, 0xe9, 0xab, 0x0f, 0xfb, 0xd5, 0xa9, 0x41, 0xbf, 0xba, 0x20, 0xaf, 0x13,
0x14, 0xa7, 0xae, 0xee, 0xc1, 0xa2, 0x71, 0x68, 0x58, 0x6d, 0x63, 0xaf, 0x8d, 0xdf, 0xb4, 0x6f,
0x38, 0x26, 0x26, 0xe5, 0x5c, 0x4d, 0x59, 0x9f, 0xdd, 0xac, 0x45, 0x39, 0x7a, 0x96, 0xa9, 0x1f,
0x9e, 0xaf, 0x7b, 0x00, 0x4d, 0xdc, 0xc6, 0x2d, 0xea, 0xb8, 0xfa, 0xf2, 0xa0, 0x5f, 0x5d, 0xbc,
0x1c, 0xc3, 0x46, 0x43, 0xf4, 0xd4, 0x06, 0xcc, 0x90, 0x03, 0xc3, 0xc5, 0xde, 0x5a, 0x39, 0x5f,
0x53, 0xd6, 0x8b, 0xfa, 0x29, 0x21, 0xe0, 0x4c, 0xd3, 0xdf, 0x40, 0x21, 0x8c, 0xf6, 0x53, 0x05,
0x56, 0xe2, 0xaa, 0xb9, 0xee, 0x98, 0xb8, 0xad, 0xde, 0x83, 0x92, 0x6d, 0x74, 0xb0, 0xe9, 0x9f,
0xcb, 0x53, 0x8f, 0x27, 0xec, 0x6b, 0xa3, 0xd5, 0x73, 0x43, 0xc2, 0x89, 0x93, 0xd6, 0xd5, 0x41,
0xbf, 0x5a, 0x92, 0x61, 0x50, 0x8c, 0x8f, 0xf6, 0xfb, 0x1c, 0x9c, 0xde, 0x72, 0xad, 0x43, 0xec,
0x0e, 0x19, 0xed, 0xc7, 0x0a, 0xac, 0x1e, 0x62, 0xdb, 0x74, 0x5c, 0x84, 0xef, 0xf6, 0x30, 0xa1,
0x3b, 0x86, 0x6b, 0x74, 0x30, 0xc5, 0xae, 0x2f, 0xde, 0xb9, 0x88, 0x78, 0x81, 0x93, 0xd4, 0xbb,
0x77, 0xf6, 0xeb, 0xc2, 0x49, 0xea, 0xc8, 0x78, 0xff, 0xf5, 0x7b, 0x14, 0xdb, 0xc4, 0x72, 0x6c,
0xbd, 0x2a, 0xb4, 0xb3, 0xba, 0x9b, 0x4c, 0x15, 0xa5, 0xb1, 0xf3, 0x44, 0x59, 0x31, 0x92, 0x34,
0x27, 0x8c, 0x7a, 0x61, 0xb4, 0x9e, 0x12, 0x95, 0xae, 0x3f, 0x2f, 0xc4, 0x49, 0xb6, 0x09, 0x4a,
0x66, 0xa8, 0xfd, 0x22, 0x07, 0x25, 0xae, 0x30, 0x21, 0x26, 0x51, 0x37, 0x01, 0x4c, 0xb6, 0xe2,
0xe9, 0x9a, 0xa9, 0x66, 0x46, 0x57, 0x05, 0x71, 0xd8, 0x0a, 0x76, 0x50, 0x04, 0x4a, 0x25, 0xb0,
0xc8, 0x0f, 0x1b, 0x51, 0x6a, 0x6e, 0x12, 0xa5, 0x96, 0x05, 0xa3, 0xc5, 0xdd, 0x18, 0x39, 0x34,
0xc4, 0x40, 0xfd, 0x26, 0x14, 0x5d, 0x21, 0x74, 0x39, 0xcf, 0xe2, 0xef, 0xdc, 0x78, 0xf1, 0x27,
0x8e, 0xaa, 0x2f, 0x0a, 0x66, 0x45, 0xff, 0xec, 0x28, 0x20, 0xa8, 0xe9, 0x50, 0x19, 0xed, 0x8f,
0x6a, 0x0d, 0x0a, 0x76, 0xa8, 0xa1, 0x39, 0x41, 0xab, 0xc0, 0x74, 0xc3, 0x76, 0xb4, 0xbf, 0x28,
0xb0, 0x1a, 0x23, 0x42, 0xa9, 0x6b, 0xed, 0xf5, 0x28, 0x3e, 0x1a, 0xdb, 0xf3, 0x92, 0x92, 0xe1,
0xc3, 0xef, 0x1a, 0xed, 0x1e, 0x16, 0x2a, 0x7d, 0x35, 0x53, 0x18, 0x49, 0x14, 0xf4, 0xcf, 0x09,
0x46, 0x6b, 0xa3, 0xa0, 0x50, 0x8c, 0xaf, 0xf6, 0x9f, 0x3c, 0x8c, 0x44, 0x50, 0xbf, 0x0d, 0xc5,
0xbb, 0x3d, 0xc3, 0xa6, 0x16, 0xbd, 0x5f, 0x3e, 0xc9, 0x84, 0xac, 0xa7, 0xda, 0x5d, 0x92, 0xfa,
0x2d, 0x81, 0xa5, 0x9f, 0x1a, 0xf4, 0xab, 0xf3, 0xfe, 0x17, 0x97, 0x22, 0x20, 0xa9, 0xbe, 0x00,
0x85, 0x3d, 0xc7, 0xe1, 0xe1, 0x51, 0xd4, 0xe7, 0xbd, 0x94, 0xa4, 0x3b, 0x4e, 0x9b, 0x83, 0xb1,
0x2d, 0xb5, 0x02, 0x79, 0xcb, 0xa6, 0xe5, 0xe9, 0x9a, 0xb2, 0x9e, 0xd7, 0xe7, 0x3c, 0xa3, 0x6e,
0xdb, 0x94, 0x03, 0x78, 0x1b, 0x6a, 0x0b, 0x8a, 0x96, 0x4d, 0x9b, 0x6d, 0xab, 0x85, 0xcb, 0x45,
0x26, 0xe1, 0x4b, 0x59, 0xd4, 0xb8, 0x2d, 0x70, 0xb9, 0x9c, 0xfe, 0x97, 0x90, 0xd3, 0x27, 0xac,
0x7e, 0x01, 0x4e, 0x12, 0xea, 0x5a, 0xf6, 0x7e, 0xf9, 0x04, 0x33, 0xeb, 0xc2, 0xa0, 0x5f, 0x9d,
0x6d, 0xb2, 0x15, 0x0e, 0x2a, 0xb6, 0x55, 0x07, 0x66, 0xf9, 0x2f, 0x2e, 0xd0, 0x0c, 0x13, 0xe8,
0x95, 0x2c, 0x02, 0x35, 0x43, 0x74, 0x9e, 0xe2, 0x23, 0x0b, 0x9c, 0x57, 0x94, 0x83, 0xfa, 0x45,
0x98, 0x3e, 0xc4, 0xae, 0x17, 0x62, 0x65, 0x60, 0xa2, 0x2d, 0x0e, 0xfa, 0xd5, 0xb9, 0x5d, 0xbe,
0xc4, 0xe1, 0x7d, 0x00, 0x6d, 0x0b, 0x96, 0x65, 0x5e, 0x57, 0xad, 0x36, 0xc5, 0xae, 0xba, 0x01,
0x45, 0x22, 0xaa, 0x8a, 0x70, 0xdb, 0x20, 0x80, 0xfc, 0x6a, 0x83, 0x02, 0x08, 0xed, 0x37, 0x0a,
0x9c, 0x8e, 0xeb, 0x90, 0x50, 0xc3, 0x6e, 0x8d, 0xe3, 0xfb, 0x16, 0x40, 0xe0, 0x82, 0x5e, 0x26,
0xf1, 0x82, 0xfb, 0xe5, 0x89, 0xdc, 0x3e, 0x4c, 0x5d, 0xc1, 0x12, 0x41, 0x11, 0xe2, 0xda, 0xc5,
0x61, 0x31, 0x85, 0x35, 0xd7, 0xa0, 0x60, 0xd9, 0x94, 0xd7, 0xf6, 0xbc, 0x5e, 0xf4, 0x44, 0xdc,
0xb6, 0x29, 0x41, 0x6c, 0x55, 0x7b, 0x1d, 0x56, 0x62, 0xc5, 0x88, 0xa7, 0x8e, 0x8c, 0x6a, 0x7a,
0x30, 0x94, 0x23, 0x82, 0x1f, 0x2a, 0x86, 0x19, 0x4b, 0xe8, 0xcc, 0xef, 0x30, 0x32, 0x3a, 0x2d,
0x47, 0x0e, 0x0b, 0xb9, 0xbf, 0x42, 0x50, 0x48, 0x59, 0xd3, 0xe1, 0x4c, 0xaa, 0x6f, 0xa9, 0x9f,
0x87, 0x69, 0xee, 0x47, 0x5c, 0x82, 0x19, 0x7d, 0x76, 0xd0, 0xaf, 0x4e, 0x73, 0x08, 0x82, 0xfc,
0x3d, 0xed, 0x8f, 0x39, 0x58, 0xde, 0x71, 0xcc, 0x66, 0xeb, 0x00, 0x9b, 0xbd, 0xb6, 0x65, 0xef,
0x5f, 0x71, 0x6c, 0x8a, 0xef, 0x51, 0xf5, 0x3d, 0x28, 0x7a, 0x4d, 0x9c, 0x69, 0x50, 0x43, 0x94,
0xd9, 0x17, 0x47, 0x65, 0x06, 0x52, 0xf7, 0xa0, 0xbd, 0x26, 0xe6, 0xcd, 0xbd, 0xef, 0xe1, 0x16,
0xbd, 0x8e, 0xa9, 0x11, 0x9a, 0x30, 0x5c, 0x43, 0x01, 0x55, 0xf5, 0x1d, 0x28, 0x90, 0x2e, 0x6e,
0x89, 0xe4, 0x78, 0x71, 0xb4, 0x82, 0x92, 0x64, 0x6c, 0x76, 0x71, 0x2b, 0xf4, 0x42, 0xef, 0x0b,
0x31, 0x8a, 0xea, 0x7b, 0x5e, 0x38, 0x1b, 0xb4, 0x47, 0x58, 0x3f, 0x34, 0xbb, 0x79, 0x69, 0x02,
0xda, 0x0c, 0x5f, 0x2f, 0x09, 0xea, 0x27, 0xf9, 0x37, 0x12, 0x74, 0xb5, 0xbf, 0x2a, 0x50, 0x4e,
0x42, 0x7b, 0xc3, 0x22, 0x54, 0xfd, 0xd6, 0x90, 0xea, 0xea, 0xe3, 0xa9, 0xce, 0xc3, 0x66, 0x8a,
0x0b, 0x1c, 0xcf, 0x5f, 0x89, 0xa8, 0xed, 0x6d, 0x38, 0x61, 0x51, 0xdc, 0xf1, 0xa3, 0x6b, 0x33,
0xfb, 0xd9, 0xf4, 0x79, 0x41, 0xfe, 0xc4, 0xb6, 0x47, 0x08, 0x71, 0x7a, 0xda, 0x87, 0x29, 0x67,
0xf2, 0x14, 0xab, 0x5e, 0x82, 0x39, 0xee, 0xfa, 0xd8, 0xf4, 0xda, 0x4e, 0x11, 0x20, 0xcb, 0x82,
0xd0, 0x5c, 0x33, 0xb2, 0x87, 0x24, 0x48, 0xf5, 0x55, 0x28, 0x75, 0x1d, 0x8a, 0x6d, 0x6a, 0x19,
0x6d, 0xbf, 0x03, 0xf6, 0xfc, 0x91, 0xb5, 0x85, 0x3b, 0xd2, 0x0e, 0x8a, 0x41, 0x6a, 0xbf, 0x52,
0xe0, 0x6c, 0xba, 0x75, 0xd4, 0xef, 0x43, 0xc9, 0x3f, 0xf1, 0x95, 0xb6, 0x61, 0x75, 0xfc, 0x60,
0xfb, 0xf2, 0x78, 0xed, 0x04, 0xc3, 0x09, 0x69, 0x0b, 0x93, 0x9f, 0x16, 0x67, 0x2a, 0x49, 0x60,
0x04, 0xc5, 0x58, 0x69, 0xbf, 0xce, 0xc1, 0xbc, 0x04, 0x72, 0x0c, 0x21, 0xf3, 0x96, 0x14, 0x32,
0x8d, 0x2c, 0xc7, 0x4c, 0x8b, 0x95, 0x5b, 0xb1, 0x58, 0x39, 0x9f, 0x85, 0xe8, 0xe8, 0x20, 0x19,
0x28, 0x50, 0x91, 0xe0, 0xaf, 0x38, 0x36, 0xe9, 0x75, 0xbc, 0x96, 0xf5, 0x36, 0x76, 0xb1, 0x57,
0x51, 0x36, 0xa0, 0x68, 0x74, 0xad, 0x6b, 0xae, 0xd3, 0xeb, 0xc6, 0x73, 0xee, 0xe5, 0x9d, 0x6d,
0xb6, 0x8e, 0x02, 0x08, 0x0f, 0xda, 0x97, 0x88, 0x49, 0x3b, 0x13, 0xed, 0x04, 0x45, 0x8b, 0x18,
0x40, 0x04, 0xd5, 0xaa, 0x90, 0x5a, 0xad, 0x74, 0xc8, 0xf7, 0x2c, 0x53, 0xd4, 0xfc, 0x17, 0x05,
0x40, 0xfe, 0xe6, 0xf6, 0xd6, 0xa7, 0xfd, 0xea, 0x0b, 0x69, 0x17, 0x4f, 0x7a, 0xbf, 0x8b, 0x49,
0xfd, 0xe6, 0xf6, 0x16, 0xf2, 0x90, 0xb5, 0x8f, 0x14, 0x38, 0x25, 0x1d, 0xf2, 0x18, 0x52, 0xc0,
0x8e, 0x9c, 0x02, 0xbe, 0x94, 0xc1, 0x64, 0x29, 0xb1, 0xff, 0xb3, 0x3c, 0xac, 0x4a, 0x70, 0x91,
0x76, 0xfd, 0xc9, 0xbb, 0xf5, 0xfb, 0x30, 0x1f, 0xdc, 0xdf, 0xaf, 0xba, 0x4e, 0x47, 0xf8, 0xf7,
0x57, 0x33, 0x9c, 0x2b, 0x72, 0xe1, 0xf0, 0x9d, 0x8b, 0xb7, 0x7c, 0xd7, 0xa2, 0x84, 0x91, 0xcc,
0x27, 0xf3, 0xdd, 0x59, 0x6d, 0x43, 0xc9, 0x94, 0x6e, 0x5d, 0xe5, 0xc2, 0x38, 0x0f, 0x08, 0xf2,
0x4d, 0x2d, 0x4c, 0x31, 0xf2, 0x3a, 0x8a, 0xd1, 0xd6, 0xfe, 0xa1, 0xc0, 0x73, 0x29, 0xa7, 0x3c,
0x06, 0x2f, 0x7b, 0x57, 0xf6, 0xb2, 0x97, 0x27, 0xb2, 0x46, 0x8a, 0xbf, 0xfd, 0x5c, 0x81, 0xda,
0x51, 0xf6, 0xcb, 0x98, 0x1c, 0x6a, 0x50, 0xb8, 0x63, 0xd9, 0x26, 0xf3, 0x9d, 0x48, 0xb8, 0x7f,
0xdd, 0xb2, 0x4d, 0xc4, 0x76, 0x82, 0x84, 0x90, 0x4f, 0xbd, 0xf8, 0x3d, 0x50, 0xe0, 0xf9, 0x91,
0xd5, 0x61, 0x8c, 0x16, 0xf8, 0x2b, 0xb0, 0xd0, 0xb3, 0x49, 0xcf, 0xa2, 0x9e, 0xc3, 0x44, 0x0b,
0xde, 0xd2, 0xa0, 0x5f, 0x5d, 0xb8, 0x29, 0x6f, 0xa1, 0x38, 0xac, 0xf6, 0xdb, 0x5c, 0x2c, 0x9f,
0xb0, 0xf2, 0x7b, 0x0d, 0x4e, 0x45, 0xca, 0x0f, 0x21, 0x91, 0x2b, 0xfe, 0x19, 0x21, 0x43, 0x14,
0x8b, 0x03, 0xa0, 0x61, 0x1c, 0x2f, 0xd4, 0xba, 0x51, 0x55, 0x7f, 0x96, 0xa1, 0x26, 0x6d, 0x20,
0x99, 0x8f, 0xba, 0x03, 0xa5, 0xf0, 0x25, 0xe3, 0xba, 0xd7, 0x42, 0x70, 0x33, 0xac, 0xfb, 0xb1,
0x70, 0x59, 0xda, 0xfd, 0x74, 0x68, 0x05, 0xc5, 0xf0, 0xb5, 0xff, 0xe6, 0x60, 0x29, 0xa1, 0x1c,
0x4d, 0xf4, 0x0e, 0xf2, 0x1d, 0x80, 0x90, 0xba, 0xd0, 0x49, 0x3d, 0xdb, 0x6b, 0x8e, 0x5e, 0x62,
0x97, 0x95, 0x70, 0x35, 0x42, 0x51, 0x25, 0x30, 0xeb, 0x62, 0x82, 0xdd, 0x43, 0x6c, 0x5e, 0x75,
0x5c, 0xf1, 0xea, 0xf1, 0x5a, 0x06, 0xa5, 0x0f, 0x95, 0x4e, 0x7d, 0x49, 0x1c, 0x69, 0x16, 0x85,
0x84, 0x51, 0x94, 0x8b, 0xda, 0x84, 0x15, 0x13, 0x47, 0x9f, 0x8f, 0x58, 0x5a, 0xc1, 0x26, 0xab,
0x88, 0xc5, 0xf0, 0xe1, 0x69, 0x2b, 0x09, 0x08, 0x25, 0xe3, 0x6a, 0x7f, 0x57, 0x60, 0x45, 0x92,
0xec, 0x1b, 0xb8, 0xd3, 0x6d, 0x1b, 0x14, 0x1f, 0x43, 0x9d, 0xb8, 0x25, 0xb5, 0x3f, 0xaf, 0x64,
0x50, 0x9f, 0x2f, 0x64, 0x5a, 0x1b, 0xa4, 0xfd, 0x4d, 0x81, 0x33, 0x89, 0x18, 0xc7, 0x90, 0x68,
0xdf, 0x91, 0x13, 0xed, 0x85, 0x09, 0xce, 0x95, 0x92, 0x66, 0x1f, 0xa5, 0x9d, 0xaa, 0xc9, 0xaf,
0x49, 0xcf, 0x5e, 0xbf, 0xaa, 0x7d, 0x9c, 0x97, 0xda, 0x6e, 0x72, 0x1c, 0xfd, 0x89, 0x9c, 0x51,
0x72, 0x63, 0x65, 0x94, 0xa1, 0x44, 0x9b, 0xcf, 0x98, 0x68, 0x09, 0x99, 0x2c, 0xd1, 0xde, 0x82,
0x79, 0xb9, 0xfa, 0x14, 0xc6, 0x1c, 0x38, 0x30, 0xd2, 0x4d, 0xa9, 0x3a, 0xc9, 0x94, 0xd4, 0x37,
0x60, 0x99, 0x50, 0xb7, 0xd7, 0xa2, 0x3d, 0x17, 0x9b, 0x91, 0x17, 0xe3, 0x13, 0x2c, 0x9f, 0x94,
0x07, 0xfd, 0xea, 0x72, 0x33, 0x61, 0x1f, 0x25, 0x62, 0xc5, 0x3b, 0x67, 0x42, 0x9e, 0xe6, 0xce,
0x99, 0xa4, 0x75, 0x32, 0x1f, 0xc9, 0x9d, 0x73, 0xd4, 0x6a, 0xcf, 0x42, 0xe7, 0x3c, 0xc2, 0xcb,
0x46, 0x76, 0xce, 0x34, 0x61, 0x70, 0xc0, 0xab, 0xda, 0x11, 0x65, 0x33, 0x3e, 0x1f, 0xc8, 0x34,
0x39, 0x78, 0x1b, 0xa6, 0x6f, 0xb3, 0x37, 0xcd, 0x31, 0xfb, 0x6e, 0xff, 0xa0, 0xfc, 0x21, 0x54,
0x5f, 0x10, 0xac, 0xa6, 0xf9, 0x37, 0x41, 0x3e, 0xb5, 0x78, 0xa7, 0x1d, 0xd5, 0xca, 0xd3, 0xdc,
0x69, 0x47, 0xe5, 0x4c, 0xf1, 0xcf, 0x3f, 0xcb, 0x9d, 0x76, 0xa2, 0xbd, 0x8f, 0xbf, 0xd3, 0xf6,
0x6e, 0x5e, 0xde, 0x5f, 0xd2, 0x35, 0x5a, 0xfe, 0x0d, 0x3d, 0xb8, 0x79, 0xdd, 0xf0, 0x37, 0x50,
0x08, 0xa3, 0x7d, 0xac, 0x40, 0x49, 0x36, 0xe7, 0x44, 0x8d, 0xde, 0x03, 0x05, 0x96, 0x5c, 0x89,
0x4c, 0x74, 0x80, 0x77, 0x3e, 0x8b, 0x3b, 0xf1, 0xf1, 0xdd, 0x73, 0x82, 0xe1, 0x52, 0xc2, 0x26,
0x4a, 0x62, 0xa5, 0xfd, 0x50, 0x81, 0x24, 0x60, 0xd5, 0x4e, 0x99, 0xbe, 0x6e, 0x66, 0x79, 0x3a,
0x16, 0x9e, 0x3e, 0xce, 0xcc, 0xf5, 0x9f, 0x11, 0x8d, 0xf2, 0x81, 0xf5, 0x44, 0x1a, 0xad, 0x41,
0x81, 0x85, 0x45, 0xcc, 0x1b, 0xb6, 0x0c, 0x6a, 0x20, 0xb6, 0xa3, 0xba, 0x50, 0x0a, 0x0b, 0x80,
0xb7, 0xce, 0x0a, 0xc6, 0x91, 0x4f, 0xbe, 0x61, 0x29, 0x89, 0xcd, 0xdf, 0xd9, 0xe1, 0x9a, 0x12,
0x45, 0x14, 0xe3, 0xa0, 0x7d, 0xa0, 0x84, 0x6d, 0x02, 0x57, 0xef, 0xdd, 0x14, 0xf5, 0x66, 0x1a,
0x4f, 0x04, 0x3f, 0xc6, 0xd2, 0xf0, 0x4f, 0x72, 0xb0, 0x10, 0x9b, 0x5d, 0x26, 0x4e, 0x5c, 0x95,
0x27, 0x3d, 0x71, 0xfd, 0x81, 0x02, 0xcb, 0xae, 0x2c, 0x48, 0xd4, 0xed, 0x37, 0x33, 0x8d, 0x5f,
0xb9, 0xdf, 0xaf, 0x09, 0xf6, 0xcb, 0x49, 0xbb, 0x28, 0x91, 0x9b, 0xf6, 0x23, 0x05, 0x12, 0xc1,
0x55, 0x27, 0xc5, 0x36, 0x17, 0xb2, 0xd9, 0x86, 0x4f, 0x87, 0xc7, 0xb1, 0xcc, 0x9f, 0x22, 0x8f,
0xb7, 0x7c, 0x5e, 0xf2, 0xe4, 0x6b, 0xf5, 0x06, 0x14, 0x6d, 0xc7, 0xc4, 0x91, 0x1e, 0x32, 0x48,
0xb2, 0x37, 0xc4, 0x3a, 0x0a, 0x20, 0x62, 0xa1, 0x98, 0x1f, 0x2b, 0x14, 0x0f, 0x60, 0xde, 0x8d,
0xfa, 0xbc, 0x68, 0xfd, 0xc6, 0xec, 0x72, 0xb8, 0x5d, 0x57, 0x04, 0x0f, 0x39, 0x7a, 0x90, 0x4c,
0x58, 0xea, 0xdd, 0x98, 0xfe, 0x9e, 0xda, 0xde, 0x8d, 0x4f, 0x5a, 0x93, 0x6b, 0xe3, 0x1f, 0xf2,
0x50, 0x4e, 0xcb, 0x32, 0xea, 0x07, 0x0a, 0xac, 0xf0, 0x40, 0x8a, 0x95, 0xcd, 0xc9, 0xc2, 0x35,
0xb8, 0x6d, 0xef, 0x26, 0xd1, 0x44, 0xc9, 0xac, 0x64, 0x21, 0xa2, 0x4f, 0x2f, 0x93, 0xfd, 0x97,
0xc6, 0xb0, 0x10, 0xd2, 0x73, 0x4e, 0x32, 0x2b, 0xc9, 0x71, 0x0b, 0x47, 0x3a, 0xee, 0x77, 0x61,
0xda, 0x65, 0x0f, 0x22, 0xde, 0xbd, 0x60, 0x8c, 0xd1, 0x67, 0xf2, 0xbf, 0xfd, 0x84, 0xbd, 0x1a,
0xff, 0x26, 0xc8, 0xa7, 0xaa, 0xfd, 0x4e, 0x81, 0xa1, 0x9c, 0x37, 0x51, 0xe5, 0x32, 0x00, 0xba,
0xff, 0xa7, 0x42, 0x03, 0x16, 0x11, 0x2d, 0x46, 0x88, 0xea, 0xfa, 0xc3, 0xc7, 0x95, 0xa9, 0x47,
0x8f, 0x2b, 0x53, 0x9f, 0x3c, 0xae, 0x4c, 0x3d, 0x18, 0x54, 0x94, 0x87, 0x83, 0x8a, 0xf2, 0x68,
0x50, 0x51, 0x3e, 0x19, 0x54, 0x94, 0x7f, 0x0d, 0x2a, 0xca, 0x87, 0xff, 0xae, 0x4c, 0xbd, 0xbb,
0x36, 0xea, 0x1f, 0x04, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x46, 0x21, 0x2d, 0x8a, 0x3f, 0x28,
0x00, 0x00,
0xd9, 0xb3, 0xbb, 0x89, 0xd7, 0x9f, 0xed, 0xb5, 0x33, 0xb6, 0xe3, 0x4d, 0xea, 0xee, 0xba, 0x23,
0x10, 0x11, 0x38, 0xbb, 0x8d, 0xd3, 0xa6, 0x51, 0x29, 0x48, 0x99, 0xb8, 0x09, 0x16, 0x6d, 0xea,
0xbe, 0x25, 0x6e, 0x53, 0xfe, 0x3a, 0xde, 0x79, 0xb1, 0x87, 0xec, 0xce, 0x6c, 0xe6, 0xbd, 0x71,
0x13, 0x71, 0x89, 0x2a, 0x10, 0x5c, 0x90, 0x8a, 0x40, 0x08, 0x4e, 0x9c, 0x38, 0x70, 0xe1, 0x02,
0x57, 0x4e, 0x15, 0x34, 0xc7, 0x20, 0x40, 0x54, 0x1c, 0x56, 0x64, 0x39, 0x72, 0xe4, 0xc6, 0x09,
0xcd, 0x7b, 0x6f, 0x7e, 0xde, 0xec, 0xcc, 0x7a, 0x67, 0x21, 0x56, 0x72, 0xf2, 0xce, 0x7b, 0xdf,
0xdf, 0xfb, 0xfe, 0xdf, 0xf7, 0x0c, 0x1b, 0x77, 0x2e, 0x93, 0x86, 0xe5, 0x34, 0x8d, 0x9e, 0xd5,
0x74, 0x31, 0x71, 0x3c, 0xb7, 0x8d, 0x9b, 0x87, 0x17, 0x8c, 0x4e, 0xef, 0xc0, 0xb8, 0xd8, 0xdc,
0xc7, 0x36, 0x76, 0x0d, 0x8a, 0xcd, 0x46, 0xcf, 0x75, 0xa8, 0xa3, 0xae, 0x71, 0xe8, 0x86, 0xd1,
0xb3, 0x1a, 0x01, 0x74, 0x23, 0x80, 0x3e, 0x7b, 0x7e, 0xdf, 0xa2, 0x07, 0xde, 0x5e, 0xa3, 0xed,
0x74, 0x9b, 0xfb, 0xce, 0xbe, 0xd3, 0x64, 0x48, 0x7b, 0xde, 0x6d, 0xf6, 0xc5, 0x3e, 0xd8, 0x2f,
0x4e, 0xec, 0xac, 0x16, 0x63, 0xdd, 0x76, 0x5c, 0x9f, 0x6d, 0x92, 0xe1, 0xd9, 0x97, 0x22, 0x98,
0xae, 0xd1, 0x3e, 0xb0, 0x6c, 0xec, 0xde, 0x6f, 0xf6, 0xee, 0xec, 0xcb, 0xf2, 0xe6, 0xc1, 0x22,
0xcd, 0x2e, 0xa6, 0x46, 0x1a, 0xaf, 0x66, 0x16, 0x96, 0xeb, 0xd9, 0xd4, 0xea, 0x0e, 0xb3, 0xb9,
0x74, 0x14, 0x02, 0x69, 0x1f, 0xe0, 0xae, 0x91, 0xc4, 0xd3, 0x7e, 0x5e, 0x80, 0xc5, 0x2b, 0x9d,
0x8e, 0xd3, 0x36, 0xa8, 0xe5, 0xd8, 0x08, 0x13, 0xaf, 0x43, 0x55, 0x07, 0x16, 0x82, 0xf3, 0x7c,
0xc5, 0xb0, 0xcd, 0x0e, 0x26, 0x55, 0x65, 0xbd, 0x78, 0x6e, 0x76, 0x73, 0xa3, 0x31, 0x4a, 0xe9,
0x0d, 0x24, 0x21, 0xe9, 0xab, 0x0f, 0xfb, 0xf5, 0xa9, 0x41, 0xbf, 0xbe, 0x20, 0xaf, 0x13, 0x94,
0xa4, 0xae, 0xee, 0xc1, 0xa2, 0x71, 0x68, 0x58, 0x1d, 0x63, 0xaf, 0x83, 0xdf, 0xb2, 0x6f, 0x38,
0x26, 0x26, 0xd5, 0xc2, 0xba, 0x72, 0x6e, 0x76, 0x73, 0x3d, 0xce, 0xd1, 0xb7, 0x4c, 0xe3, 0xf0,
0x42, 0xc3, 0x07, 0x68, 0xe1, 0x0e, 0x6e, 0x53, 0xc7, 0xd5, 0x97, 0x07, 0xfd, 0xfa, 0xe2, 0x95,
0x04, 0x36, 0x1a, 0xa2, 0xa7, 0x36, 0x61, 0x86, 0x1c, 0x18, 0x2e, 0xf6, 0xd7, 0xaa, 0xc5, 0x75,
0xe5, 0x5c, 0x59, 0x3f, 0x25, 0x04, 0x9c, 0x69, 0x05, 0x1b, 0x28, 0x82, 0xd1, 0x7e, 0xac, 0xc0,
0x4a, 0x52, 0x35, 0x6f, 0x3a, 0x26, 0xee, 0xa8, 0xf7, 0xa0, 0x62, 0x1b, 0x5d, 0x6c, 0x06, 0xe7,
0xf2, 0xd5, 0xe3, 0x0b, 0xfb, 0xda, 0x68, 0xf5, 0xdc, 0x90, 0x70, 0x92, 0xa4, 0x75, 0x75, 0xd0,
0xaf, 0x57, 0x64, 0x18, 0x94, 0xe0, 0xa3, 0xfd, 0xa6, 0x00, 0xa7, 0xb7, 0x5c, 0xeb, 0x10, 0xbb,
0x43, 0x46, 0xfb, 0xa1, 0x02, 0xab, 0x87, 0xd8, 0x36, 0x1d, 0x17, 0xe1, 0xbb, 0x1e, 0x26, 0x74,
0xc7, 0x70, 0x8d, 0x2e, 0xa6, 0xd8, 0x0d, 0xc4, 0x3b, 0x1f, 0x13, 0x2f, 0x74, 0x92, 0x46, 0xef,
0xce, 0x7e, 0x43, 0x38, 0x49, 0x03, 0x19, 0x1f, 0xbc, 0x7e, 0x8f, 0x62, 0x9b, 0x58, 0x8e, 0xad,
0xd7, 0x85, 0x76, 0x56, 0x77, 0xd3, 0xa9, 0xa2, 0x2c, 0x76, 0xbe, 0x28, 0x2b, 0x46, 0x9a, 0xe6,
0x84, 0x51, 0x2f, 0x8e, 0xd6, 0x53, 0xaa, 0xd2, 0xf5, 0xe7, 0x85, 0x38, 0xe9, 0x36, 0x41, 0xe9,
0x0c, 0xb5, 0x9f, 0x15, 0xa0, 0xc2, 0x15, 0x26, 0xc4, 0x24, 0xea, 0x26, 0x80, 0xc9, 0x56, 0x7c,
0x5d, 0x33, 0xd5, 0xcc, 0xe8, 0xaa, 0x20, 0x0e, 0x5b, 0xe1, 0x0e, 0x8a, 0x41, 0xa9, 0x04, 0x16,
0xf9, 0x61, 0x63, 0x4a, 0x2d, 0x4c, 0xa2, 0xd4, 0xaa, 0x60, 0xb4, 0xb8, 0x9b, 0x20, 0x87, 0x86,
0x18, 0xa8, 0x5f, 0x87, 0xb2, 0x2b, 0x84, 0xae, 0x16, 0x59, 0xfc, 0x9d, 0x1f, 0x2f, 0xfe, 0xc4,
0x51, 0xf5, 0x45, 0xc1, 0xac, 0x1c, 0x9c, 0x1d, 0x85, 0x04, 0x35, 0x1d, 0x6a, 0xa3, 0xfd, 0x51,
0x5d, 0x87, 0x92, 0x1d, 0x69, 0x68, 0x4e, 0xd0, 0x2a, 0x31, 0xdd, 0xb0, 0x1d, 0xed, 0x8f, 0x0a,
0xac, 0x26, 0x88, 0x50, 0xea, 0x5a, 0x7b, 0x1e, 0xc5, 0x47, 0x63, 0xfb, 0x5e, 0x52, 0x31, 0x02,
0xf8, 0x5d, 0xa3, 0xe3, 0x61, 0xa1, 0xd2, 0x57, 0x73, 0x85, 0x91, 0x44, 0x41, 0xff, 0x8c, 0x60,
0xb4, 0x36, 0x0a, 0x0a, 0x25, 0xf8, 0x6a, 0xff, 0x2a, 0xc2, 0x48, 0x04, 0xf5, 0x9b, 0x50, 0xbe,
0xeb, 0x19, 0x36, 0xb5, 0xe8, 0xfd, 0xea, 0x49, 0x26, 0x64, 0x23, 0xd3, 0xee, 0x92, 0xd4, 0x6f,
0x0b, 0x2c, 0xfd, 0xd4, 0xa0, 0x5f, 0x9f, 0x0f, 0xbe, 0xb8, 0x14, 0x21, 0x49, 0xf5, 0x05, 0x28,
0xed, 0x39, 0x0e, 0x0f, 0x8f, 0xb2, 0x3e, 0xef, 0xa7, 0x24, 0xdd, 0x71, 0x3a, 0x1c, 0x8c, 0x6d,
0xa9, 0x35, 0x28, 0x5a, 0x36, 0xad, 0x4e, 0xaf, 0x2b, 0xe7, 0x8a, 0xfa, 0x9c, 0x6f, 0xd4, 0x6d,
0x9b, 0x72, 0x00, 0x7f, 0x43, 0x6d, 0x43, 0xd9, 0xb2, 0x69, 0xab, 0x63, 0xb5, 0x71, 0xb5, 0xcc,
0x24, 0x7c, 0x29, 0x8f, 0x1a, 0xb7, 0x05, 0x2e, 0x97, 0x33, 0xf8, 0x12, 0x72, 0x06, 0x84, 0xd5,
0xcf, 0xc1, 0x49, 0x42, 0x5d, 0xcb, 0xde, 0xaf, 0x9e, 0x60, 0x66, 0x5d, 0x18, 0xf4, 0xeb, 0xb3,
0x2d, 0xb6, 0xc2, 0x41, 0xc5, 0xb6, 0xea, 0xc0, 0x2c, 0xff, 0xc5, 0x05, 0x9a, 0x61, 0x02, 0xbd,
0x92, 0x47, 0xa0, 0x56, 0x84, 0xce, 0x53, 0x7c, 0x6c, 0x81, 0xf3, 0x8a, 0x73, 0x50, 0x3f, 0x0f,
0xd3, 0x87, 0xd8, 0xf5, 0x43, 0xac, 0x0a, 0x4c, 0xb4, 0xc5, 0x41, 0xbf, 0x3e, 0xb7, 0xcb, 0x97,
0x38, 0x7c, 0x00, 0xa0, 0x6d, 0xc1, 0xb2, 0xcc, 0xeb, 0x9a, 0xd5, 0xa1, 0xd8, 0x55, 0x37, 0xa0,
0x4c, 0x44, 0x55, 0x11, 0x6e, 0x1b, 0x06, 0x50, 0x50, 0x6d, 0x50, 0x08, 0xa1, 0xfd, 0x4a, 0x81,
0xd3, 0x49, 0x1d, 0x12, 0x6a, 0xd8, 0xed, 0x71, 0x7c, 0xdf, 0x02, 0x08, 0x5d, 0xd0, 0xcf, 0x24,
0x7e, 0x70, 0xbf, 0x3c, 0x91, 0xdb, 0x47, 0xa9, 0x2b, 0x5c, 0x22, 0x28, 0x46, 0x5c, 0xbb, 0x34,
0x2c, 0xa6, 0xb0, 0xe6, 0x1a, 0x94, 0x2c, 0x9b, 0xf2, 0xda, 0x5e, 0xd4, 0xcb, 0xbe, 0x88, 0xdb,
0x36, 0x25, 0x88, 0xad, 0x6a, 0xaf, 0xc3, 0x4a, 0xa2, 0x18, 0xf1, 0xd4, 0x91, 0x53, 0x4d, 0x0f,
0x86, 0x72, 0x44, 0xf8, 0x43, 0xc5, 0x30, 0x63, 0x09, 0x9d, 0x05, 0x1d, 0x46, 0x4e, 0xa7, 0xe5,
0xc8, 0x51, 0x21, 0x0f, 0x56, 0x08, 0x8a, 0x28, 0x6b, 0x3a, 0x9c, 0xc9, 0xf4, 0x2d, 0xf5, 0xb3,
0x30, 0xcd, 0xfd, 0x88, 0x4b, 0x30, 0xa3, 0xcf, 0x0e, 0xfa, 0xf5, 0x69, 0x0e, 0x41, 0x50, 0xb0,
0xa7, 0xfd, 0xae, 0x00, 0xcb, 0x3b, 0x8e, 0xd9, 0x6a, 0x1f, 0x60, 0xd3, 0xeb, 0x58, 0xf6, 0xfe,
0x55, 0xc7, 0xa6, 0xf8, 0x1e, 0x55, 0xdf, 0x87, 0xb2, 0xdf, 0xc4, 0x99, 0x06, 0x35, 0x44, 0x99,
0x7d, 0x71, 0x54, 0x66, 0x20, 0x0d, 0x1f, 0xda, 0x6f, 0x62, 0xde, 0xda, 0xfb, 0x0e, 0x6e, 0xd3,
0x37, 0x31, 0x35, 0x22, 0x13, 0x46, 0x6b, 0x28, 0xa4, 0xaa, 0xbe, 0x0b, 0x25, 0xd2, 0xc3, 0x6d,
0x91, 0x1c, 0x2f, 0x8d, 0x56, 0x50, 0x9a, 0x8c, 0xad, 0x1e, 0x6e, 0x47, 0x5e, 0xe8, 0x7f, 0x21,
0x46, 0x51, 0x7d, 0xdf, 0x0f, 0x67, 0x83, 0x7a, 0x84, 0xf5, 0x43, 0xb3, 0x9b, 0x97, 0x27, 0xa0,
0xcd, 0xf0, 0xf5, 0x8a, 0xa0, 0x7e, 0x92, 0x7f, 0x23, 0x41, 0x57, 0xfb, 0x93, 0x02, 0xd5, 0x34,
0xb4, 0x37, 0x2c, 0x42, 0xd5, 0x6f, 0x0c, 0xa9, 0xae, 0x31, 0x9e, 0xea, 0x7c, 0x6c, 0xa6, 0xb8,
0xd0, 0xf1, 0x82, 0x95, 0x98, 0xda, 0xde, 0x81, 0x13, 0x16, 0xc5, 0xdd, 0x20, 0xba, 0x36, 0xf3,
0x9f, 0x4d, 0x9f, 0x17, 0xe4, 0x4f, 0x6c, 0xfb, 0x84, 0x10, 0xa7, 0xa7, 0x7d, 0x94, 0x71, 0x26,
0x5f, 0xb1, 0xea, 0x65, 0x98, 0xe3, 0xae, 0x8f, 0x4d, 0xbf, 0xed, 0x14, 0x01, 0xb2, 0x2c, 0x08,
0xcd, 0xb5, 0x62, 0x7b, 0x48, 0x82, 0x54, 0x5f, 0x85, 0x4a, 0xcf, 0xa1, 0xd8, 0xa6, 0x96, 0xd1,
0x09, 0x3a, 0x60, 0xdf, 0x1f, 0x59, 0x5b, 0xb8, 0x23, 0xed, 0xa0, 0x04, 0xa4, 0xf6, 0x0b, 0x05,
0xce, 0x66, 0x5b, 0x47, 0xfd, 0x2e, 0x54, 0x82, 0x13, 0x5f, 0xed, 0x18, 0x56, 0x37, 0x08, 0xb6,
0x2f, 0x8e, 0xd7, 0x4e, 0x30, 0x9c, 0x88, 0xb6, 0x30, 0xf9, 0x69, 0x71, 0xa6, 0x8a, 0x04, 0x46,
0x50, 0x82, 0x95, 0xf6, 0xcb, 0x02, 0xcc, 0x4b, 0x20, 0xc7, 0x10, 0x32, 0x6f, 0x4b, 0x21, 0xd3,
0xcc, 0x73, 0xcc, 0xac, 0x58, 0xb9, 0x95, 0x88, 0x95, 0x0b, 0x79, 0x88, 0x8e, 0x0e, 0x92, 0x81,
0x02, 0x35, 0x09, 0xfe, 0xaa, 0x63, 0x13, 0xaf, 0xeb, 0xb7, 0xac, 0xb7, 0xb1, 0x8b, 0xfd, 0x8a,
0xb2, 0x01, 0x65, 0xa3, 0x67, 0x5d, 0x77, 0x1d, 0xaf, 0x97, 0xcc, 0xb9, 0x57, 0x76, 0xb6, 0xd9,
0x3a, 0x0a, 0x21, 0x7c, 0xe8, 0x40, 0x22, 0x26, 0xed, 0x4c, 0xbc, 0x13, 0x14, 0x2d, 0x62, 0x08,
0x11, 0x56, 0xab, 0x52, 0x66, 0xb5, 0xd2, 0xa1, 0xe8, 0x59, 0xa6, 0xa8, 0xf9, 0x2f, 0x0a, 0x80,
0xe2, 0xcd, 0xed, 0xad, 0xff, 0xf4, 0xeb, 0x2f, 0x64, 0x5d, 0x3c, 0xe9, 0xfd, 0x1e, 0x26, 0x8d,
0x9b, 0xdb, 0x5b, 0xc8, 0x47, 0xd6, 0x3e, 0x56, 0xe0, 0x94, 0x74, 0xc8, 0x63, 0x48, 0x01, 0x3b,
0x72, 0x0a, 0xf8, 0x42, 0x0e, 0x93, 0x65, 0xc4, 0xfe, 0x4f, 0x8a, 0xb0, 0x2a, 0xc1, 0xc5, 0xda,
0xf5, 0x27, 0xef, 0xd6, 0x1f, 0xc0, 0x7c, 0x78, 0x7f, 0xbf, 0xe6, 0x3a, 0x5d, 0xe1, 0xdf, 0x5f,
0xce, 0x71, 0xae, 0xd8, 0x85, 0x23, 0x70, 0x2e, 0xde, 0xf2, 0x5d, 0x8f, 0x13, 0x46, 0x32, 0x9f,
0xdc, 0x77, 0x67, 0xb5, 0x03, 0x15, 0x53, 0xba, 0x75, 0x55, 0x4b, 0xe3, 0x0c, 0x10, 0xe4, 0x9b,
0x5a, 0x94, 0x62, 0xe4, 0x75, 0x94, 0xa0, 0xad, 0xfd, 0x4d, 0x81, 0xe7, 0x32, 0x4e, 0x79, 0x0c,
0x5e, 0xf6, 0x9e, 0xec, 0x65, 0x2f, 0x4f, 0x64, 0x8d, 0x0c, 0x7f, 0xfb, 0xa9, 0x02, 0xeb, 0x47,
0xd9, 0x2f, 0x67, 0x72, 0x58, 0x87, 0xd2, 0x1d, 0xcb, 0x36, 0x99, 0xef, 0xc4, 0xc2, 0xfd, 0xab,
0x96, 0x6d, 0x22, 0xb6, 0x13, 0x26, 0x84, 0x62, 0xe6, 0xc5, 0xef, 0x81, 0x02, 0xcf, 0x8f, 0xac,
0x0e, 0x63, 0xb4, 0xc0, 0x5f, 0x82, 0x05, 0xcf, 0x26, 0x9e, 0x45, 0x7d, 0x87, 0x89, 0x17, 0xbc,
0xa5, 0x41, 0xbf, 0xbe, 0x70, 0x53, 0xde, 0x42, 0x49, 0x58, 0xed, 0xaf, 0xc9, 0x7c, 0xc2, 0xca,
0xef, 0x75, 0x38, 0x15, 0x2b, 0x3f, 0x84, 0xc4, 0xae, 0xf8, 0x67, 0x84, 0x0c, 0x71, 0x2c, 0x0e,
0x80, 0x86, 0x71, 0xfc, 0x50, 0xeb, 0xc5, 0x55, 0xfd, 0xff, 0x0c, 0x35, 0x69, 0x03, 0xc9, 0x7c,
0xb4, 0x7f, 0x17, 0x60, 0x29, 0xa5, 0x78, 0x4c, 0x34, 0xb5, 0xf8, 0x16, 0x40, 0x34, 0x15, 0x11,
0x27, 0x68, 0xe4, 0x9b, 0xbd, 0xe8, 0x15, 0x76, 0xb5, 0x88, 0x56, 0x63, 0x14, 0x55, 0x02, 0xb3,
0x2e, 0x26, 0xd8, 0x3d, 0xc4, 0xe6, 0x35, 0xc7, 0x15, 0x33, 0x8a, 0xd7, 0x72, 0xa8, 0x68, 0xa8,
0xd0, 0xe9, 0x4b, 0xe2, 0x48, 0xb3, 0x28, 0x22, 0x8c, 0xe2, 0x5c, 0xd4, 0x16, 0xac, 0x98, 0x38,
0x3e, 0xec, 0x61, 0x49, 0x00, 0x9b, 0xac, 0x7e, 0x95, 0xa3, 0x31, 0xd1, 0x56, 0x1a, 0x10, 0x4a,
0xc7, 0xd5, 0xfe, 0xa2, 0xc0, 0x8a, 0x24, 0xd9, 0xd7, 0x70, 0xb7, 0xd7, 0x31, 0x28, 0x3e, 0x86,
0xac, 0x7e, 0x4b, 0x6a, 0x56, 0x5e, 0xc9, 0xa1, 0xbe, 0x40, 0xc8, 0xac, 0xa6, 0x45, 0xfb, 0xb3,
0x02, 0x67, 0x52, 0x31, 0x8e, 0x21, 0x2d, 0xbe, 0x2b, 0xa7, 0xc5, 0x8b, 0x13, 0x9c, 0x2b, 0x23,
0x29, 0x3e, 0xca, 0x3a, 0x55, 0x8b, 0x5f, 0x6a, 0x9e, 0xbd, 0xee, 0x52, 0xfb, 0xa4, 0x28, 0x35,
0xc9, 0xe4, 0x38, 0xba, 0x09, 0x39, 0xa3, 0x14, 0xc6, 0xca, 0x28, 0x43, 0x69, 0xb1, 0x98, 0x33,
0x2d, 0x12, 0x32, 0x51, 0x5a, 0x54, 0x6f, 0xc1, 0xbc, 0x5c, 0x2b, 0x4a, 0x63, 0x3e, 0x0f, 0x30,
0xd2, 0x2d, 0xa9, 0x96, 0xc8, 0x94, 0xd4, 0x37, 0x60, 0x99, 0x50, 0xd7, 0x6b, 0x53, 0xcf, 0xc5,
0x66, 0x6c, 0xbe, 0x7b, 0x82, 0xe5, 0x93, 0xea, 0xa0, 0x5f, 0x5f, 0x6e, 0xa5, 0xec, 0xa3, 0x54,
0xac, 0x64, 0x9f, 0x4b, 0xc8, 0xd3, 0xdc, 0xe7, 0x92, 0xac, 0xbe, 0xe3, 0x63, 0xb9, 0xcf, 0x8d,
0x5b, 0xed, 0x59, 0xe8, 0x73, 0x47, 0x78, 0xd9, 0xc8, 0x3e, 0x97, 0xa6, 0x8c, 0xf9, 0x79, 0x55,
0x3b, 0xa2, 0x6c, 0x26, 0xa7, 0xf9, 0xb9, 0xe6, 0xfc, 0xef, 0xc0, 0xf4, 0x6d, 0x36, 0x81, 0x1c,
0xb3, 0x4b, 0x0e, 0x0e, 0xca, 0xc7, 0x96, 0xfa, 0x82, 0x60, 0x35, 0xcd, 0xbf, 0x09, 0x0a, 0xa8,
0x25, 0xfb, 0xe2, 0xb8, 0x56, 0x9e, 0xe6, 0xbe, 0x38, 0x2e, 0x67, 0x86, 0x7f, 0xfe, 0x41, 0xee,
0x8b, 0x53, 0xed, 0x7d, 0xfc, 0x7d, 0xb1, 0x7f, 0x4f, 0xf2, 0xff, 0x92, 0x9e, 0xd1, 0x0e, 0xee,
0xd3, 0xe1, 0x3d, 0xe9, 0x46, 0xb0, 0x81, 0x22, 0x18, 0xed, 0x13, 0x05, 0x2a, 0xb2, 0x39, 0x27,
0x6a, 0xf4, 0x1e, 0x28, 0xb0, 0xe4, 0x4a, 0x64, 0xe2, 0xcf, 0x6d, 0x17, 0xf2, 0xb8, 0x13, 0x7f,
0x6c, 0x7b, 0x4e, 0x30, 0x5c, 0x4a, 0xd9, 0x44, 0x69, 0xac, 0xb4, 0xef, 0x2b, 0x90, 0x06, 0xac,
0xda, 0x19, 0x6f, 0xa5, 0x9b, 0x79, 0x06, 0xbd, 0xc2, 0xd3, 0xc7, 0x79, 0x21, 0xfd, 0x7b, 0x4c,
0xa3, 0xfc, 0x79, 0x79, 0x22, 0x8d, 0xae, 0x43, 0x89, 0x85, 0x45, 0xc2, 0x1b, 0xb6, 0x0c, 0x6a,
0x20, 0xb6, 0xa3, 0xba, 0x50, 0x89, 0x0a, 0x80, 0xbf, 0xce, 0x0a, 0xc6, 0x91, 0x03, 0xda, 0xa8,
0x94, 0x24, 0x5e, 0xcb, 0xd9, 0xe1, 0x5a, 0x12, 0x45, 0x94, 0xe0, 0xa0, 0x7d, 0xa8, 0x44, 0x6d,
0x02, 0x57, 0xef, 0xdd, 0x0c, 0xf5, 0xe6, 0x7a, 0x4c, 0x08, 0x7f, 0x8c, 0xa5, 0xe1, 0x1f, 0x15,
0x60, 0x21, 0xf1, 0xd2, 0x98, 0xfa, 0x3e, 0xaa, 0x3c, 0xe9, 0xf7, 0xd1, 0xef, 0x29, 0xb0, 0xec,
0xca, 0x82, 0xc4, 0xdd, 0x7e, 0x33, 0xd7, 0x63, 0x29, 0xf7, 0xfb, 0x35, 0xc1, 0x7e, 0x39, 0x6d,
0x17, 0xa5, 0x72, 0xd3, 0x7e, 0xa0, 0x40, 0x2a, 0xb8, 0xea, 0x64, 0xd8, 0xe6, 0x62, 0x3e, 0xdb,
0xf0, 0xb7, 0xdc, 0x71, 0x2c, 0xf3, 0xfb, 0xd8, 0xa8, 0x95, 0xbf, 0x6e, 0x3c, 0xf9, 0x5a, 0xbd,
0x01, 0x65, 0xdb, 0x31, 0x71, 0xac, 0x87, 0x0c, 0x93, 0xec, 0x0d, 0xb1, 0x8e, 0x42, 0x88, 0x44,
0x28, 0x16, 0xc7, 0x0a, 0xc5, 0x03, 0x98, 0x77, 0xe3, 0x3e, 0x2f, 0x5a, 0xbf, 0x31, 0xbb, 0x1c,
0x6e, 0xd7, 0x15, 0xc1, 0x43, 0x8e, 0x1e, 0x24, 0x13, 0x96, 0x7a, 0x37, 0xa6, 0xbf, 0xa7, 0xb6,
0x77, 0xe3, 0xef, 0xa2, 0xe9, 0xb5, 0xf1, 0xb7, 0x45, 0xa8, 0x66, 0x65, 0x19, 0xf5, 0x43, 0x05,
0x56, 0x78, 0x20, 0x25, 0xca, 0xe6, 0x64, 0xe1, 0x1a, 0xde, 0xb6, 0x77, 0xd3, 0x68, 0xa2, 0x74,
0x56, 0xb2, 0x10, 0xf1, 0x41, 0xc9, 0x64, 0xff, 0x53, 0x31, 0x2c, 0x84, 0x34, 0x7c, 0x49, 0x67,
0x25, 0x39, 0x6e, 0xe9, 0x48, 0xc7, 0xfd, 0x36, 0x4c, 0xbb, 0x6c, 0x20, 0xe2, 0xdf, 0x0b, 0xc6,
0x78, 0xa8, 0x4c, 0xff, 0x27, 0x9d, 0xa8, 0x57, 0xe3, 0xdf, 0x04, 0x05, 0x54, 0xb5, 0x5f, 0x2b,
0x30, 0x94, 0xf3, 0x26, 0xaa, 0x5c, 0x06, 0x40, 0xef, 0x7f, 0x54, 0x68, 0xc8, 0x22, 0xa6, 0xc5,
0x18, 0x51, 0x5d, 0x7f, 0xf8, 0xb8, 0x36, 0xf5, 0xe8, 0x71, 0x6d, 0xea, 0xd3, 0xc7, 0xb5, 0xa9,
0x07, 0x83, 0x9a, 0xf2, 0x70, 0x50, 0x53, 0x1e, 0x0d, 0x6a, 0xca, 0xa7, 0x83, 0x9a, 0xf2, 0x8f,
0x41, 0x4d, 0xf9, 0xe8, 0x9f, 0xb5, 0xa9, 0xf7, 0xd6, 0x46, 0xfd, 0x3b, 0xdf, 0x7f, 0x03, 0x00,
0x00, 0xff, 0xff, 0xf0, 0x19, 0xbb, 0x6c, 0xed, 0x27, 0x00, 0x00,
}
func (m *AllocationResult) Marshal() (dAtA []byte, err error) {
@@ -2539,11 +2537,6 @@ func (m *ResourceClaimSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
i -= len(m.AllocationMode)
copy(dAtA[i:], m.AllocationMode)
i = encodeVarintGenerated(dAtA, i, uint64(len(m.AllocationMode)))
i--
dAtA[i] = 0x1a
if m.ParametersRef != nil {
{
size, err := m.ParametersRef.MarshalToSizedBuffer(dAtA[:i])
@@ -3886,8 +3879,6 @@ func (m *ResourceClaimSpec) Size() (n int) {
l = m.ParametersRef.Size()
n += 1 + l + sovGenerated(uint64(l))
}
l = len(m.AllocationMode)
n += 1 + l + sovGenerated(uint64(l))
return n
}
@@ -4536,7 +4527,6 @@ func (this *ResourceClaimSpec) String() string {
s := strings.Join([]string{`&ResourceClaimSpec{`,
`ResourceClassName:` + fmt.Sprintf("%v", this.ResourceClassName) + `,`,
`ParametersRef:` + strings.Replace(this.ParametersRef.String(), "ResourceClaimParametersReference", "ResourceClaimParametersReference", 1) + `,`,
`AllocationMode:` + fmt.Sprintf("%v", this.AllocationMode) + `,`,
`}`,
}, "")
return s
@@ -7889,38 +7879,6 @@ func (m *ResourceClaimSpec) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field AllocationMode", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.AllocationMode = AllocationMode(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:])

View File

@@ -411,11 +411,6 @@ message ResourceClaimSpec {
// The object must be in the same namespace as the ResourceClaim.
// +optional
optional ResourceClaimParametersReference parametersRef = 2;
// Allocation can start immediately or when a Pod wants to use the
// resource. "WaitForFirstConsumer" is the default.
// +optional
optional string allocationMode = 3;
}
// ResourceClaimStatus tracks whether the resource has been allocated and what

View File

@@ -72,34 +72,8 @@ type ResourceClaimSpec struct {
// The object must be in the same namespace as the ResourceClaim.
// +optional
ParametersRef *ResourceClaimParametersReference `json:"parametersRef,omitempty" protobuf:"bytes,2,opt,name=parametersRef"`
// Allocation can start immediately or when a Pod wants to use the
// resource. "WaitForFirstConsumer" is the default.
// +optional
AllocationMode AllocationMode `json:"allocationMode,omitempty" protobuf:"bytes,3,opt,name=allocationMode"`
}
// AllocationMode describes whether a ResourceClaim gets allocated immediately
// when it gets created (AllocationModeImmediate) or whether allocation is
// delayed until it is needed for a Pod
// (AllocationModeWaitForFirstConsumer). Other modes might get added in the
// future.
type AllocationMode string
const (
// When a ResourceClaim has AllocationModeWaitForFirstConsumer, allocation is
// delayed until a Pod gets scheduled that needs the ResourceClaim. The
// scheduler will consider all resource requirements of that Pod and
// trigger allocation for a node that fits the Pod.
AllocationModeWaitForFirstConsumer AllocationMode = "WaitForFirstConsumer"
// When a ResourceClaim has AllocationModeImmediate, allocation starts
// as soon as the ResourceClaim gets created. This is done without
// considering the needs of Pods that will use the ResourceClaim
// because those Pods are not known yet.
AllocationModeImmediate AllocationMode = "Immediate"
)
// ResourceClaimStatus tracks whether the resource has been allocated and what
// the resulting attributes are.
type ResourceClaimStatus struct {

View File

@@ -187,7 +187,6 @@ var map_ResourceClaimSpec = map[string]string{
"": "ResourceClaimSpec defines how a resource is to be allocated.",
"resourceClassName": "ResourceClassName references the driver and additional parameters via the name of a ResourceClass that was created as part of the driver deployment.",
"parametersRef": "ParametersRef references a separate object with arbitrary parameters that will be used by the driver when allocating a resource for the claim.\n\nThe object must be in the same namespace as the ResourceClaim.",
"allocationMode": "Allocation can start immediately or when a Pod wants to use the resource. \"WaitForFirstConsumer\" is the default.",
}
func (ResourceClaimSpec) SwaggerDoc() map[string]string {

View File

@@ -49,8 +49,7 @@
"apiGroup": "apiGroupValue",
"kind": "kindValue",
"name": "nameValue"
},
"allocationMode": "allocationModeValue"
}
},
"status": {
"driverName": "driverNameValue",

View File

@@ -33,7 +33,6 @@ metadata:
selfLink: selfLinkValue
uid: uidValue
spec:
allocationMode: allocationModeValue
parametersRef:
apiGroup: apiGroupValue
kind: kindValue

View File

@@ -92,8 +92,7 @@
"apiGroup": "apiGroupValue",
"kind": "kindValue",
"name": "nameValue"
},
"allocationMode": "allocationModeValue"
}
}
}
}

View File

@@ -66,7 +66,6 @@ spec:
selfLink: selfLinkValue
uid: uidValue
spec:
allocationMode: allocationModeValue
parametersRef:
apiGroup: apiGroupValue
kind: kindValue

View File

@@ -12419,9 +12419,6 @@ var schemaYAML = typed.YAMLObject(`types:
- name: io.k8s.api.resource.v1alpha3.ResourceClaimSpec
map:
fields:
- name: allocationMode
type:
scalar: string
- name: parametersRef
type:
namedType: io.k8s.api.resource.v1alpha3.ResourceClaimParametersReference

View File

@@ -18,16 +18,11 @@ limitations under the License.
package v1alpha3
import (
resourcev1alpha3 "k8s.io/api/resource/v1alpha3"
)
// ResourceClaimSpecApplyConfiguration represents a declarative configuration of the ResourceClaimSpec type for use
// with apply.
type ResourceClaimSpecApplyConfiguration struct {
ResourceClassName *string `json:"resourceClassName,omitempty"`
ParametersRef *ResourceClaimParametersReferenceApplyConfiguration `json:"parametersRef,omitempty"`
AllocationMode *resourcev1alpha3.AllocationMode `json:"allocationMode,omitempty"`
}
// ResourceClaimSpecApplyConfiguration constructs a declarative configuration of the ResourceClaimSpec type for use with
@@ -51,11 +46,3 @@ func (b *ResourceClaimSpecApplyConfiguration) WithParametersRef(value *ResourceC
b.ParametersRef = value
return b
}
// WithAllocationMode sets the AllocationMode field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the AllocationMode field is set to the value of the last call.
func (b *ResourceClaimSpecApplyConfiguration) WithAllocationMode(value resourcev1alpha3.AllocationMode) *ResourceClaimSpecApplyConfiguration {
b.AllocationMode = &value
return b
}

View File

@@ -515,48 +515,7 @@ func (ctrl *controller) syncClaim(ctx context.Context, claim *resourceapi.Resour
logger.V(5).Info("ResourceClaim is allocated")
return nil
}
if claim.Spec.AllocationMode != resourceapi.AllocationModeImmediate {
logger.V(5).Info("ResourceClaim waiting for first consumer")
return nil
}
// We need the ResourceClass to determine whether we should allocate it.
class, err := ctrl.rcLister.Get(claim.Spec.ResourceClassName)
if err != nil {
return err
}
if class.DriverName != ctrl.name {
// Not ours *at the moment*. This can change, so requeue and
// check again. We could trigger a faster check when the
// ResourceClass changes, but that shouldn't occur much in
// practice and thus isn't worth the effort.
//
// We use exponential backoff because it is unlikely that
// the ResourceClass changes much.
logger.V(5).Info("ResourceClaim is handled by other driver", "driver", class.DriverName)
return errRequeue
}
// Check parameters. Do not record event to Claim if its parameters are invalid,
// syncKey will record the error.
claimParameters, classParameters, err := ctrl.getParameters(ctx, claim, class, false)
if err != nil {
return err
}
claimAllocations := claimAllocations{&ClaimAllocation{
Claim: claim,
ClaimParameters: claimParameters,
Class: class,
ClassParameters: classParameters,
}}
ctrl.allocateClaims(ctx, claimAllocations, "", nil)
if claimAllocations[0].Error != nil {
return fmt.Errorf("allocate: %v", claimAllocations[0].Error)
}
logger.V(5).Info("ResourceClaim waiting for first consumer")
return nil
}
@@ -678,10 +637,6 @@ func (ctrl *controller) checkPodClaim(ctx context.Context, pod *v1.Pod, podClaim
return nil, err
}
}
if claim.Spec.AllocationMode != resourceapi.AllocationModeWaitForFirstConsumer {
// Nothing to do for it as part of pod scheduling.
return nil, nil
}
if claim.Status.Allocation != nil {
// Already allocated, class and parameter are not needed and nothing
// need to be done for the claim either.

View File

@@ -53,8 +53,6 @@ func TestController(t *testing.T) {
}
claim := createClaim(claimName, claimNamespace, className)
otherClaim := createClaim(claimName, claimNamespace, otherClassName)
delayedClaim := claim.DeepCopy()
delayedClaim.Spec.AllocationMode = resourceapi.AllocationModeWaitForFirstConsumer
podName := "pod"
podKey := "schedulingCtx:default/pod"
pod := createPod(podName, claimNamespace, nil)
@@ -148,94 +146,6 @@ func TestController(t *testing.T) {
classes: classes,
claim: otherClaim,
expectedClaim: otherClaim,
expectedError: errRequeue.Error(), // class might change
},
// Immediate allocation:
// deletion time stamp set, our finalizer set, not allocated -> remove finalizer
"immediate-deleted-finalizer-removal": {
key: claimKey,
classes: classes,
claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer),
driver: m.expectDeallocate(map[string]error{claimName: nil}),
expectedClaim: withDeletionTimestamp(claim),
},
// deletion time stamp set, our finalizer set, not allocated, stopping fails -> requeue
"immediate-deleted-finalizer-stop-failure": {
key: claimKey,
classes: classes,
claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer),
driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}),
expectedClaim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer),
expectedError: "stop allocation: fake error",
},
// deletion time stamp set, other finalizer set, not allocated -> do nothing
"immediate-deleted-finalizer-no-removal": {
key: claimKey,
classes: classes,
claim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer),
expectedClaim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer),
},
// deletion time stamp set, finalizer set, allocated -> deallocate
"immediate-deleted-allocated": {
key: claimKey,
classes: classes,
claim: withAllocate(withDeletionTimestamp(claim)),
driver: m.expectDeallocate(map[string]error{claimName: nil}),
expectedClaim: withDeletionTimestamp(claim),
},
// deletion time stamp set, finalizer set, allocated, deallocation fails -> requeue
"immediate-deleted-deallocate-failure": {
key: claimKey,
classes: classes,
claim: withAllocate(withDeletionTimestamp(claim)),
driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}),
expectedClaim: withAllocate(withDeletionTimestamp(claim)),
expectedError: "deallocate: fake error",
},
// deletion time stamp set, finalizer not set -> do nothing
"immediate-deleted-no-finalizer": {
key: claimKey,
classes: classes,
claim: withDeletionTimestamp(claim),
expectedClaim: withDeletionTimestamp(claim),
},
// not deleted, not allocated, no finalizer -> add finalizer, allocate
"immediate-do-allocation": {
key: claimKey,
classes: classes,
claim: claim,
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
expectClaimParameters(map[string]interface{}{claimName: 2}).
expectAllocate(map[string]allocate{claimName: {allocResult: &allocation, allocErr: nil}}),
expectedClaim: withAllocate(claim),
},
// not deleted, not allocated, finalizer -> allocate
"immediate-continue-allocation": {
key: claimKey,
classes: classes,
claim: withFinalizer(claim, ourFinalizer),
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
expectClaimParameters(map[string]interface{}{claimName: 2}).
expectAllocate(map[string]allocate{claimName: {allocResult: &allocation, allocErr: nil}}),
expectedClaim: withAllocate(claim),
},
// not deleted, not allocated, finalizer, fail allocation -> requeue
"immediate-fail-allocation": {
key: claimKey,
classes: classes,
claim: withFinalizer(claim, ourFinalizer),
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
expectClaimParameters(map[string]interface{}{claimName: 2}).
expectAllocate(map[string]allocate{claimName: {allocErr: errors.New("fake error")}}),
expectedClaim: withFinalizer(claim, ourFinalizer),
expectedError: "allocate: fake error",
},
// not deleted, allocated -> do nothing
"immediate-allocated-nop": {
key: claimKey,
classes: classes,
claim: withAllocate(claim),
expectedClaim: withAllocate(claim),
},
// not deleted, reallocate -> deallocate
@@ -257,62 +167,60 @@ func TestController(t *testing.T) {
expectedError: "deallocate: fake error",
},
// Delayed allocation is similar in some cases, but not quite
// the same.
// deletion time stamp set, our finalizer set, not allocated -> remove finalizer
"delayed-deleted-finalizer-removal": {
"deleted-finalizer-removal": {
key: claimKey,
classes: classes,
claim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer),
claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer),
driver: m.expectDeallocate(map[string]error{claimName: nil}),
expectedClaim: withDeletionTimestamp(delayedClaim),
expectedClaim: withDeletionTimestamp(claim),
},
// deletion time stamp set, our finalizer set, not allocated, stopping fails -> requeue
"delayed-deleted-finalizer-stop-failure": {
"deleted-finalizer-stop-failure": {
key: claimKey,
classes: classes,
claim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer),
claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer),
driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}),
expectedClaim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer),
expectedClaim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer),
expectedError: "stop allocation: fake error",
},
// deletion time stamp set, other finalizer set, not allocated -> do nothing
"delayed-deleted-finalizer-no-removal": {
"deleted-finalizer-no-removal": {
key: claimKey,
classes: classes,
claim: withFinalizer(withDeletionTimestamp(delayedClaim), otherFinalizer),
expectedClaim: withFinalizer(withDeletionTimestamp(delayedClaim), otherFinalizer),
claim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer),
expectedClaim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer),
},
// deletion time stamp set, finalizer set, allocated -> deallocate
"delayed-deleted-allocated": {
"deleted-allocated": {
key: claimKey,
classes: classes,
claim: withAllocate(withDeletionTimestamp(delayedClaim)),
claim: withAllocate(withDeletionTimestamp(claim)),
driver: m.expectDeallocate(map[string]error{claimName: nil}),
expectedClaim: withDeletionTimestamp(delayedClaim),
expectedClaim: withDeletionTimestamp(claim),
},
// deletion time stamp set, finalizer set, allocated, deallocation fails -> requeue
"delayed-deleted-deallocate-failure": {
"deleted-deallocate-failure": {
key: claimKey,
classes: classes,
claim: withAllocate(withDeletionTimestamp(delayedClaim)),
claim: withAllocate(withDeletionTimestamp(claim)),
driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}),
expectedClaim: withAllocate(withDeletionTimestamp(delayedClaim)),
expectedClaim: withAllocate(withDeletionTimestamp(claim)),
expectedError: "deallocate: fake error",
},
// deletion time stamp set, finalizer not set -> do nothing
"delayed-deleted-no-finalizer": {
"deleted-no-finalizer": {
key: claimKey,
classes: classes,
claim: withDeletionTimestamp(delayedClaim),
expectedClaim: withDeletionTimestamp(delayedClaim),
claim: withDeletionTimestamp(claim),
expectedClaim: withDeletionTimestamp(claim),
},
// waiting for first consumer -> do nothing
"delayed-pending": {
"pending": {
key: claimKey,
classes: classes,
claim: delayedClaim,
expectedClaim: delayedClaim,
claim: claim,
expectedClaim: claim,
},
// pod with no claims -> shouldn't occur, check again anyway
@@ -324,34 +232,23 @@ func TestController(t *testing.T) {
expectedError: errPeriodic.Error(),
},
// pod with immediate allocation and selected node -> shouldn't occur, check again in case that claim changes
"pod-immediate": {
key: podKey,
claim: claim,
expectedClaim: claim,
pod: podWithClaim,
schedulingCtx: withSelectedNode(podSchedulingCtx),
expectedSchedulingCtx: withSelectedNode(podSchedulingCtx),
expectedError: errPeriodic.Error(),
},
// pod with delayed allocation, no potential nodes -> shouldn't occur
"pod-delayed-no-nodes": {
// no potential nodes -> shouldn't occur
"no-nodes": {
key: podKey,
classes: classes,
claim: delayedClaim,
expectedClaim: delayedClaim,
claim: claim,
expectedClaim: claim,
pod: podWithClaim,
schedulingCtx: podSchedulingCtx,
expectedSchedulingCtx: podSchedulingCtx,
},
// pod with delayed allocation, potential nodes -> provide unsuitable nodes
"pod-delayed-info": {
// potential nodes -> provide unsuitable nodes
"info": {
key: podKey,
classes: classes,
claim: delayedClaim,
expectedClaim: delayedClaim,
claim: claim,
expectedClaim: claim,
pod: podWithClaim,
schedulingCtx: withPotentialNodes(podSchedulingCtx),
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
@@ -361,23 +258,23 @@ func TestController(t *testing.T) {
expectedError: errPeriodic.Error(),
},
// pod with delayed allocation, potential nodes, selected node, missing class -> failure
"pod-delayed-missing-class": {
// potential nodes, selected node, missing class -> failure
"missing-class": {
key: podKey,
claim: delayedClaim,
expectedClaim: delayedClaim,
claim: claim,
expectedClaim: claim,
pod: podWithClaim,
schedulingCtx: withSelectedNode(withPotentialNodes(podSchedulingCtx)),
expectedSchedulingCtx: withSelectedNode(withPotentialNodes(podSchedulingCtx)),
expectedError: `pod claim my-pod-claim: resourceclass.resource.k8s.io "mock-class" not found`,
},
// pod with delayed allocation, potential nodes, selected node -> allocate
"pod-delayed-allocate": {
// potential nodes, selected node -> allocate
"allocate": {
key: podKey,
classes: classes,
claim: delayedClaim,
expectedClaim: withReservedFor(withAllocate(delayedClaim), pod),
claim: claim,
expectedClaim: withReservedFor(withAllocate(claim), pod),
pod: podWithClaim,
schedulingCtx: withSelectedNode(withPotentialNodes(podSchedulingCtx)),
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
@@ -387,12 +284,12 @@ func TestController(t *testing.T) {
expectedSchedulingCtx: withUnsuitableNodes(withSelectedNode(withPotentialNodes(podSchedulingCtx))),
expectedError: errPeriodic.Error(),
},
// pod with delayed allocation, potential nodes, selected node, all unsuitable -> update unsuitable nodes
"pod-selected-is-potential-node": {
// potential nodes, selected node, all unsuitable -> update unsuitable nodes
"is-potential-node": {
key: podKey,
classes: classes,
claim: delayedClaim,
expectedClaim: delayedClaim,
claim: claim,
expectedClaim: claim,
pod: podWithClaim,
schedulingCtx: withPotentialNodes(withSelectedNode(withPotentialNodes(podSchedulingCtx))),
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
@@ -401,12 +298,12 @@ func TestController(t *testing.T) {
expectedSchedulingCtx: withSpecificUnsuitableNodes(withSelectedNode(withPotentialNodes(podSchedulingCtx)), potentialNodes),
expectedError: errPeriodic.Error(),
},
// pod with delayed allocation, max potential nodes, other selected node, all unsuitable -> update unsuitable nodes with truncation at start
"pod-selected-is-potential-node-truncate-first": {
// max potential nodes, other selected node, all unsuitable -> update unsuitable nodes with truncation at start
"is-potential-node-truncate-first": {
key: podKey,
classes: classes,
claim: delayedClaim,
expectedClaim: delayedClaim,
claim: claim,
expectedClaim: claim,
pod: podWithClaim,
schedulingCtx: withSpecificPotentialNodes(withSelectedNode(withSpecificPotentialNodes(podSchedulingCtx, maxNodes)), maxNodes),
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
@@ -415,12 +312,12 @@ func TestController(t *testing.T) {
expectedSchedulingCtx: withSpecificUnsuitableNodes(withSelectedNode(withSpecificPotentialNodes(podSchedulingCtx, maxNodes)), append(maxNodes[1:], nodeName)),
expectedError: errPeriodic.Error(),
},
// pod with delayed allocation, max potential nodes, other selected node, all unsuitable (but in reverse order) -> update unsuitable nodes with truncation at end
// max potential nodes, other selected node, all unsuitable (but in reverse order) -> update unsuitable nodes with truncation at end
"pod-selected-is-potential-node-truncate-last": {
key: podKey,
classes: classes,
claim: delayedClaim,
expectedClaim: delayedClaim,
claim: claim,
expectedClaim: claim,
pod: podWithClaim,
schedulingCtx: withSpecificPotentialNodes(withSelectedNode(withSpecificPotentialNodes(podSchedulingCtx, maxNodes)), maxNodes),
driver: m.expectClassParameters(map[string]interface{}{className: 1}).
@@ -652,7 +549,6 @@ func createClaim(claimName, claimNamespace, className string) *resourceapi.Resou
},
Spec: resourceapi.ResourceClaimSpec{
ResourceClassName: className,
AllocationMode: resourceapi.AllocationModeImmediate,
},
}
}

View File

@@ -27,7 +27,6 @@ import (
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gcustom"
"github.com/onsi/gomega/gstruct"
"github.com/onsi/gomega/types"
@@ -100,7 +99,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.By("waiting for container startup to fail")
parameters := b.parameters()
pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, template := b.podInline()
b.create(ctx, parameters, pod, template)
@@ -126,7 +125,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
// Pretend that the resource is allocated and reserved for some other entity.
// Until the resourceclaim controller learns to remove reservations for
// arbitrary types we can simply fake somthing here.
claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim := b.externalClaim()
b.create(ctx, claim)
claim, err := f.ClientSet.ResourceV1alpha3().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
@@ -180,7 +179,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("must unprepare resources for force-deleted pod", func(ctx context.Context) {
parameters := b.parameters()
claim := b.externalClaim(resourceapi.AllocationModeImmediate)
claim := b.externalClaim()
pod := b.podExternal()
zero := int64(0)
pod.Spec.TerminationGracePeriodSeconds = &zero
@@ -203,7 +202,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("must skip NodePrepareResource if not used by any container", func(ctx context.Context) {
parameters := b.parameters()
pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, template := b.podInline()
for i := range pod.Spec.Containers {
pod.Spec.Containers[i].Resources.Claims = nil
}
@@ -219,10 +218,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
// claimTests tries out several different combinations of pods with
// claims, both inline and external.
claimTests := func(b *builder, driver *Driver, allocationMode resourceapi.AllocationMode) {
claimTests := func(b *builder, driver *Driver) {
ginkgo.It("supports simple pod referencing inline resource claim", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod, template := b.podInline(allocationMode)
pod, template := b.podInline()
objects = append(objects, pod, template)
b.create(ctx, objects...)
@@ -231,7 +230,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("supports inline claim referenced by multiple containers", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod, template := b.podInlineMultiple(allocationMode)
pod, template := b.podInlineMultiple()
objects = append(objects, pod, template)
b.create(ctx, objects...)
@@ -241,7 +240,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod := b.podExternal()
claim := b.externalClaim(allocationMode)
claim := b.externalClaim()
objects = append(objects, claim, pod)
b.create(ctx, objects...)
@@ -253,7 +252,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
pod1 := b.podExternal()
pod2 := b.podExternal()
pod3 := b.podExternal()
claim := b.externalClaim(allocationMode)
claim := b.externalClaim()
objects = append(objects, claim, pod1, pod2, pod3)
b.create(ctx, objects...)
@@ -267,7 +266,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
pod1 := b.podExternalMultiple()
pod2 := b.podExternalMultiple()
pod3 := b.podExternalMultiple()
claim := b.externalClaim(allocationMode)
claim := b.externalClaim()
objects = append(objects, claim, pod1, pod2, pod3)
b.create(ctx, objects...)
@@ -278,7 +277,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("supports init containers", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod, template := b.podInline(allocationMode)
pod, template := b.podInline()
pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]}
pod.Spec.InitContainers[0].Name += "-init"
// This must succeed for the pod to start.
@@ -292,7 +291,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) {
objects, _ := b.flexibleParameters()
pod := b.podExternal()
claim := b.externalClaim(allocationMode)
claim := b.externalClaim()
pod.Spec.Containers[0].Command = []string{"true"}
objects = append(objects, claim, pod)
b.create(ctx, objects...)
@@ -307,7 +306,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("deletes generated claims when pod is done", func(ctx context.Context) {
objects, _ := b.flexibleParameters()
pod, template := b.podInline(allocationMode)
pod, template := b.podInline()
pod.Spec.Containers[0].Command = []string{"true"}
objects = append(objects, template, pod)
b.create(ctx, objects...)
@@ -326,7 +325,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("does not delete generated claims when pod is restarting", func(ctx context.Context) {
objects, _ := b.flexibleParameters()
pod, template := b.podInline(allocationMode)
pod, template := b.podInline()
pod.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 1; exit 1"}
pod.Spec.RestartPolicy = v1.RestartPolicyAlways
objects = append(objects, template, pod)
@@ -341,10 +340,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
}
})
ginkgo.It("must deallocate after use when using delayed allocation", func(ctx context.Context) {
ginkgo.It("must deallocate after use", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod := b.podExternal()
claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim := b.externalClaim()
objects = append(objects, claim, pod)
b.create(ctx, objects...)
@@ -383,7 +382,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("supports claim and class parameters", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, template := b.podInline()
objects = append(objects, pod, template)
b.create(ctx, objects...)
@@ -395,7 +394,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
objects, expectedEnv := b.flexibleParameters()
pods := make([]*v1.Pod, numPods)
for i := 0; i < numPods; i++ {
pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, template := b.podInline()
pods[i] = pod
objects = append(objects, pod, template)
}
@@ -421,7 +420,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
objects = append(objects, b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer))
objects = append(objects, b.externalClaim())
pods := make([]*v1.Pod, numPods)
for i := 0; i < numPods; i++ {
@@ -459,7 +458,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
objects[len(objects)-1].(*resourceapi.ResourceClaimParameters).Shareable = false
}
objects = append(objects, b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer))
objects = append(objects, b.externalClaim())
pods := make([]*v1.Pod, numPods)
for i := 0; i < numPods; i++ {
@@ -491,7 +490,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("retries pod scheduling after creating resource class", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, template := b.podInline()
class, err := f.ClientSet.ResourceV1alpha3().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{})
framework.ExpectNoError(err)
template.Spec.Spec.ResourceClassName += "-b"
@@ -510,7 +509,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("retries pod scheduling after updating resource class", func(ctx context.Context) {
objects, expectedEnv := b.flexibleParameters()
pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, template := b.podInline()
// First modify the class so that it matches no nodes.
class, err := f.ClientSet.ResourceV1alpha3().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{})
@@ -546,7 +545,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
})
ginkgo.It("runs a pod without a generated resource claim", func(ctx context.Context) {
pod, _ /* template */ := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, _ /* template */ := b.podInline()
created := b.create(ctx, pod)
pod = created[0].(*v1.Pod)
@@ -563,13 +562,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod))
})
ginkgo.Context("with delayed allocation", func() {
claimTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer)
})
ginkgo.Context("with immediate allocation", func() {
claimTests(b, driver, resourceapi.AllocationModeImmediate)
})
claimTests(b, driver)
}
// These tests depend on having more than one node and a DRA driver controller.
@@ -603,7 +596,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
}
pod1 := createPod()
pod2 := createPod()
claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim := b.externalClaim()
b.create(ctx, parameters, claim, pod1, pod2)
for _, pod := range []*v1.Pod{pod1, pod2} {
@@ -624,7 +617,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
instance := f.UniqueName + "-test-app"
pod := b.podExternal()
pod.Labels[label] = instance
claim := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim := b.externalClaim()
b.create(ctx, parameters, claim, pod)
ginkgo.By("wait for test pod " + pod.Name + " to run")
@@ -714,13 +707,13 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
parameters1 := b.parameters()
parameters2 := b2.parameters()
// Order is relevant here: each pod must be matched with its own claim.
pod1claim1 := b.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
pod1claim1 := b.externalClaim()
pod1 := b.podExternal()
pod2claim1 := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
pod2claim1 := b2.externalClaim()
pod2 := b2.podExternal()
// Add another claim to pod1.
pod1claim2 := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
pod1claim2 := b2.externalClaim()
pod1.Spec.ResourceClaims = append(pod1.Spec.ResourceClaims,
v1.PodResourceClaim{
Name: "claim-other",
@@ -801,51 +794,41 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
driver.parameterMode = parameterMode
b := newBuilder(f, driver)
tests := func(allocationMode resourceapi.AllocationMode) {
ginkgo.It("uses all resources", func(ctx context.Context) {
objs, _ := b.flexibleParameters()
var pods []*v1.Pod
for i := 0; i < len(nodes.NodeNames); i++ {
pod, template := b.podInline(allocationMode)
pods = append(pods, pod)
objs = append(objs, pod, template)
ginkgo.It("uses all resources", func(ctx context.Context) {
objs, _ := b.flexibleParameters()
var pods []*v1.Pod
for i := 0; i < len(nodes.NodeNames); i++ {
pod, template := b.podInline()
pods = append(pods, pod)
objs = append(objs, pod, template)
}
b.create(ctx, objs...)
for _, pod := range pods {
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
}
// The pods all should run on different
// nodes because the maximum number of
// claims per node was limited to 1 for
// this test.
//
// We cannot know for sure why the pods
// ran on two different nodes (could
// also be a coincidence) but if they
// don't cover all nodes, then we have
// a problem.
used := make(map[string]*v1.Pod)
for _, pod := range pods {
pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "get pod")
nodeName := pod.Spec.NodeName
if other, ok := used[nodeName]; ok {
framework.Failf("Pod %s got started on the same node %s as pod %s although claim allocation should have been limited to one claim per node.", pod.Name, nodeName, other.Name)
}
b.create(ctx, objs...)
for _, pod := range pods {
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
}
// The pods all should run on different
// nodes because the maximum number of
// claims per node was limited to 1 for
// this test.
//
// We cannot know for sure why the pods
// ran on two different nodes (could
// also be a coincidence) but if they
// don't cover all nodes, then we have
// a problem.
used := make(map[string]*v1.Pod)
for _, pod := range pods {
pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "get pod")
nodeName := pod.Spec.NodeName
if other, ok := used[nodeName]; ok {
framework.Failf("Pod %s got started on the same node %s as pod %s although claim allocation should have been limited to one claim per node.", pod.Name, nodeName, other.Name)
}
used[nodeName] = pod
}
})
}
ginkgo.Context("with delayed allocation", func() {
tests(resourceapi.AllocationModeWaitForFirstConsumer)
})
ginkgo.Context("with immediate allocation", func() {
tests(resourceapi.AllocationModeImmediate)
used[nodeName] = pod
}
})
})
}
@@ -873,7 +856,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("truncates the name of a generated resource claim", func(ctx context.Context) {
parameters := b.parameters()
pod, template := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
pod, template := b.podInline()
pod.Name = strings.Repeat("p", 63)
pod.Spec.ResourceClaims[0].Name = strings.Repeat("c", 63)
pod.Spec.Containers[0].Resources.Claims[0].Name = pod.Spec.ResourceClaims[0].Name
@@ -1039,118 +1022,16 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
// which is the goal for 1.31 to support version skew for kubelet.
})
ginkgo.Context("with local unshared resources", func() {
driver := NewDriver(f, nodes, func() app.Resources {
return app.Resources{
NodeLocal: true,
MaxAllocations: 10,
Nodes: nodes.NodeNames,
}
})
b := newBuilder(f, driver)
// This test covers some special code paths in the scheduler:
// - Patching the ReservedFor during PreBind because in contrast
// to claims specifically allocated for a pod, here the claim
// gets allocated without reserving it.
// - Error handling when PreBind fails: multiple attempts to bind pods
// are started concurrently, only one attempt succeeds.
// - Removing a ReservedFor entry because the first inline claim gets
// reserved during allocation.
ginkgo.It("reuses an allocated immediate claim", func(ctx context.Context) {
objects := []klog.KMetadata{
b.parameters(),
b.externalClaim(resourceapi.AllocationModeImmediate),
}
podExternal := b.podExternal()
// Create many pods to increase the chance that the scheduler will
// try to bind two pods at the same time.
numPods := 5
for i := 0; i < numPods; i++ {
podInline, claimTemplate := b.podInline(resourceapi.AllocationModeWaitForFirstConsumer)
podInline.Spec.Containers[0].Resources.Claims = append(podInline.Spec.Containers[0].Resources.Claims, podExternal.Spec.Containers[0].Resources.Claims[0])
podInline.Spec.ResourceClaims = append(podInline.Spec.ResourceClaims, podExternal.Spec.ResourceClaims[0])
objects = append(objects, claimTemplate, podInline)
}
b.create(ctx, objects...)
var runningPod *v1.Pod
haveRunningPod := gcustom.MakeMatcher(func(pods []v1.Pod) (bool, error) {
numRunning := 0
runningPod = nil
for _, pod := range pods {
if pod.Status.Phase == v1.PodRunning {
pod := pod // Don't keep pointer to loop variable...
runningPod = &pod
numRunning++
}
}
return numRunning == 1, nil
}).WithTemplate("Expected one running Pod.\nGot instead:\n{{.FormattedActual}}")
for i := 0; i < numPods; i++ {
ginkgo.By("waiting for exactly one pod to start")
runningPod = nil
gomega.Eventually(ctx, b.listTestPods).WithTimeout(f.Timeouts.PodStartSlow).Should(haveRunningPod)
ginkgo.By("checking that no other pod gets scheduled")
havePendingPods := gcustom.MakeMatcher(func(pods []v1.Pod) (bool, error) {
numPending := 0
for _, pod := range pods {
if pod.Status.Phase == v1.PodPending {
numPending++
}
}
return numPending == numPods-1-i, nil
}).WithTemplate("Expected only one running Pod.\nGot instead:\n{{.FormattedActual}}")
gomega.Consistently(ctx, b.listTestPods).WithTimeout(time.Second).Should(havePendingPods)
ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(runningPod)))
framework.ExpectNoError(b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, runningPod.Name, metav1.DeleteOptions{}))
ginkgo.By(fmt.Sprintf("waiting for pod %s to disappear", klog.KObj(runningPod)))
framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, b.f.ClientSet, runningPod.Name, runningPod.Namespace, f.Timeouts.PodDelete))
}
})
})
ginkgo.Context("with shared network resources", func() {
driver := NewDriver(f, nodes, networkResources)
b := newBuilder(f, driver)
// This test complements "reuses an allocated immediate claim" above:
// because the claim can be shared, each PreBind attempt succeeds.
ginkgo.It("shares an allocated immediate claim", func(ctx context.Context) {
objects := []klog.KMetadata{
b.parameters(),
b.externalClaim(resourceapi.AllocationModeImmediate),
}
// Create many pods to increase the chance that the scheduler will
// try to bind two pods at the same time.
numPods := 5
pods := make([]*v1.Pod, numPods)
for i := 0; i < numPods; i++ {
pods[i] = b.podExternal()
objects = append(objects, pods[i])
}
b.create(ctx, objects...)
ginkgo.By("waiting all pods to start")
framework.ExpectNoError(e2epod.WaitForPodsRunning(ctx, b.f.ClientSet, f.Namespace.Name, numPods+len(nodes.NodeNames) /* driver(s) */, f.Timeouts.PodStartSlow))
})
})
// kube-controller-manager can trigger delayed allocation for pods where the
// node name was already selected when creating the pod. For immediate
// allocation, the creator has to ensure that the node matches the claims.
// This does not work for resource claim templates and only isn't
// a problem here because the resource is network-attached and available
// on all nodes.
preScheduledTests := func(b *builder, driver *Driver, allocationMode resourceapi.AllocationMode) {
preScheduledTests := func(b *builder, driver *Driver) {
ginkgo.It("supports scheduled pod referencing inline resource claim", func(ctx context.Context) {
parameters := b.parameters()
pod, template := b.podInline(allocationMode)
pod, template := b.podInline()
pod.Spec.NodeName = nodes.NodeNames[0]
b.create(ctx, parameters, pod, template)
@@ -1159,7 +1040,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("supports scheduled pod referencing external resource claim", func(ctx context.Context) {
parameters := b.parameters()
claim := b.externalClaim(allocationMode)
claim := b.externalClaim()
pod := b.podExternal()
pod.Spec.NodeName = nodes.NodeNames[0]
b.create(ctx, parameters, claim, pod)
@@ -1168,30 +1049,22 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
})
}
ginkgo.Context("with delayed allocation and setting ReservedFor", func() {
ginkgo.Context("with setting ReservedFor", func() {
driver := NewDriver(f, nodes, networkResources)
b := newBuilder(f, driver)
preScheduledTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer)
claimTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer)
preScheduledTests(b, driver)
claimTests(b, driver)
})
ginkgo.Context("with delayed allocation and not setting ReservedFor", func() {
ginkgo.Context("without setting ReservedFor", func() {
driver := NewDriver(f, nodes, func() app.Resources {
resources := networkResources()
resources.DontSetReservedFor = true
return resources
})
b := newBuilder(f, driver)
preScheduledTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer)
claimTests(b, driver, resourceapi.AllocationModeWaitForFirstConsumer)
})
ginkgo.Context("with immediate allocation", func() {
driver := NewDriver(f, nodes, networkResources)
b := newBuilder(f, driver)
preScheduledTests(b, driver, resourceapi.AllocationModeImmediate)
claimTests(b, driver, resourceapi.AllocationModeImmediate)
preScheduledTests(b, driver)
claimTests(b, driver)
})
})
@@ -1209,10 +1082,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("work", func(ctx context.Context) {
parameters1 := b1.parameters()
parameters2 := b2.parameters()
claim1 := b1.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim1b := b1.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim2 := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim2b := b2.externalClaim(resourceapi.AllocationModeWaitForFirstConsumer)
claim1 := b1.externalClaim()
claim1b := b1.externalClaim()
claim2 := b2.externalClaim()
claim2b := b2.externalClaim()
pod := b1.podExternal()
for i, claim := range []*resourceapi.ResourceClaim{claim1b, claim2, claim2b} {
claim := claim
@@ -1298,7 +1171,7 @@ func (b *builder) nodeSelector() *v1.NodeSelector {
// externalClaim returns external resource claim
// that test pods can reference
func (b *builder) externalClaim(allocationMode resourceapi.AllocationMode) *resourceapi.ResourceClaim {
func (b *builder) externalClaim() *resourceapi.ResourceClaim {
b.claimCounter++
name := "external-claim" + b.driver.NameSuffix // This is what podExternal expects.
if b.claimCounter > 1 {
@@ -1315,7 +1188,6 @@ func (b *builder) externalClaim(allocationMode resourceapi.AllocationMode) *reso
Kind: b.driver.claimParameterAPIKind,
Name: b.parametersName(),
},
AllocationMode: allocationMode,
},
}
}
@@ -1493,7 +1365,7 @@ func (b *builder) pod() *v1.Pod {
}
// makePodInline adds an inline resource claim with default class name and parameters.
func (b *builder) podInline(allocationMode resourceapi.AllocationMode) (*v1.Pod, *resourceapi.ResourceClaimTemplate) {
func (b *builder) podInline() (*v1.Pod, *resourceapi.ResourceClaimTemplate) {
pod := b.pod()
pod.Spec.Containers[0].Name = "with-resource"
podClaimName := "my-inline-claim"
@@ -1517,7 +1389,6 @@ func (b *builder) podInline(allocationMode resourceapi.AllocationMode) (*v1.Pod,
Kind: b.driver.claimParameterAPIKind,
Name: b.parametersName(),
},
AllocationMode: allocationMode,
},
},
}
@@ -1525,8 +1396,8 @@ func (b *builder) podInline(allocationMode resourceapi.AllocationMode) (*v1.Pod,
}
// podInlineMultiple returns a pod with inline resource claim referenced by 3 containers
func (b *builder) podInlineMultiple(allocationMode resourceapi.AllocationMode) (*v1.Pod, *resourceapi.ResourceClaimTemplate) {
pod, template := b.podInline(allocationMode)
func (b *builder) podInlineMultiple() (*v1.Pod, *resourceapi.ResourceClaimTemplate) {
pod, template := b.podInline()
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy(), *pod.Spec.Containers[0].DeepCopy())
pod.Spec.Containers[1].Name = pod.Spec.Containers[1].Name + "-1"
pod.Spec.Containers[2].Name = pod.Spec.Containers[1].Name + "-2"

View File

@@ -78,8 +78,8 @@ configmap/pause-claim-parameters created
pod/pause created
$ kubectl get resourceclaims
NAME CLASSNAME ALLOCATIONMODE STATE AGE
pause-resource example WaitForFirstConsumer allocated,reserved 19s
NAME CLASSNAME STATE AGE
pause-resource example allocated,reserved 19s
$ kubectl get pods
NAME READY STATUS RESTARTS AGE

View File

@@ -12,7 +12,6 @@ metadata:
name: example
namespace: default
spec:
allocationMode: Immediate
resourceClassName: example
parametersRef:
kind: ConfigMap

View File

@@ -409,11 +409,11 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes
ExpectedEtcdPath: "/registry/resourceclasses/class1name",
},
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): {
Stub: `{"metadata": {"name": "claim1name"}, "spec": {"resourceClassName": "class1name", "allocationMode": "WaitForFirstConsumer"}}`,
Stub: `{"metadata": {"name": "claim1name"}, "spec": {"resourceClassName": "class1name"}}`,
ExpectedEtcdPath: "/registry/resourceclaims/" + namespace + "/claim1name",
},
gvr("resource.k8s.io", "v1alpha3", "resourceclaimtemplates"): {
Stub: `{"metadata": {"name": "claimtemplate1name"}, "spec": {"spec": {"resourceClassName": "class1name", "allocationMode": "WaitForFirstConsumer"}}}`,
Stub: `{"metadata": {"name": "claimtemplate1name"}, "spec": {"spec": {"resourceClassName": "class1name"}}}`,
ExpectedEtcdPath: "/registry/resourceclaimtemplates/" + namespace + "/claimtemplate1name",
},
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): {