Extend the Job API for BackoffLimitPerIndex

2023-07-13 09:36:07 +02:00
parent b2a9c06b2e
commit fcbfdc1710
31 changed files with 1359 additions and 133 deletions
--- a/pkg/apis/batch/validation/validation_test.go
+++ b/pkg/apis/batch/validation/validation_test.go
@@ -160,6 +160,28 @@ func TestValidateJob(t *testing.T) {
 				},
 			},
 		},
+		"valid pod failure policy with FailIndex": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Completions:          pointer.Int32(2),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					Selector:             validGeneratedSelector,
+					ManualSelector:       pointer.Bool(true),
+					Template:             validPodTemplateSpecForGeneratedRestartPolicyNever,
+					PodFailurePolicy: &batch.PodFailurePolicy{
+						Rules: []batch.PodFailurePolicyRule{{
+							Action: batch.PodFailurePolicyActionFailIndex,
+							OnExitCodes: &batch.PodFailurePolicyOnExitCodesRequirement{
+								Operator: batch.PodFailurePolicyOnExitCodesOpIn,
+								Values:   []int32{10},
+							},
+						}},
+					},
+				},
+			},
+		},
 		"valid manual selector": {
 			opts: JobValidationOptions{RequirePrefixedLabels: true},
 			job: batch.Job{
@@ -236,6 +258,36 @@ func TestValidateJob(t *testing.T) {
 				},
 			},
 		},
+		"valid parallelism and maxFailedIndexes for high completions when backoffLimitPerIndex is used": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					Completions:          pointer.Int32(100_000),
+					Parallelism:          pointer.Int32(100_000),
+					MaxFailedIndexes:     pointer.Int32(100_000),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"valid parallelism and maxFailedIndexes for unlimited completions when backoffLimitPerIndex is used": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					Completions:          pointer.Int32(1_000_000_000),
+					Parallelism:          pointer.Int32(10_000),
+					MaxFailedIndexes:     pointer.Int32(10_000),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
 		"valid job tracking annotation": {
 			opts: JobValidationOptions{
 				RequirePrefixedLabels: true,
@@ -476,7 +528,7 @@ func TestValidateJob(t *testing.T) {
 			},
 			opts: JobValidationOptions{RequirePrefixedLabels: true},
 		},
-		`spec.podFailurePolicy.rules[0].action: Required value: valid values: ["Count" "FailJob" "Ignore"]`: {
+		`spec.podFailurePolicy.rules[0].action: Required value: valid values: ["Count" "FailIndex" "FailJob" "Ignore"]`: {
 			job: batch.Job{
 				ObjectMeta: validJobObjectMeta,
 				Spec: batch.JobSpec{
@@ -584,7 +636,7 @@ func TestValidateJob(t *testing.T) {
 			},
 			opts: JobValidationOptions{RequirePrefixedLabels: true},
 		},
-		`spec.podFailurePolicy.rules[0].action: Unsupported value: "UnknownAction": supported values: "Count", "FailJob", "Ignore"`: {
+		`spec.podFailurePolicy.rules[0].action: Unsupported value: "UnknownAction": supported values: "Count", "FailIndex", "FailJob", "Ignore"`: {
 			job: batch.Job{
 				ObjectMeta: validJobObjectMeta,
 				Spec: batch.JobSpec{
@@ -749,6 +801,124 @@ func TestValidateJob(t *testing.T) {
 			},
 			opts: JobValidationOptions{RequirePrefixedLabels: true},
 		},
+		"spec.backoffLimitPerIndex: Invalid value: 1: requires indexed completion mode": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					BackoffLimitPerIndex: pointer.Int32(1),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.backoffLimitPerIndex:must be greater than or equal to 0": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					BackoffLimitPerIndex: pointer.Int32(-1),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.maxFailedIndexes: Invalid value: 11: must be less than or equal to completions": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					Completions:          pointer.Int32(10),
+					MaxFailedIndexes:     pointer.Int32(11),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.maxFailedIndexes: Required value: must be specified when completions is above 100000": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					Completions:          pointer.Int32(100_001),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.parallelism: Invalid value: 50000: must be less than or equal to 10000 when completions are above 100000 and used with backoff limit per index": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					Completions:          pointer.Int32(100_001),
+					Parallelism:          pointer.Int32(50_000),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					MaxFailedIndexes:     pointer.Int32(1),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.maxFailedIndexes: Invalid value: 100001: must be less than or equal to 100000": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					Completions:          pointer.Int32(100_001),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					MaxFailedIndexes:     pointer.Int32(100_001),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.maxFailedIndexes: Invalid value: 50000: must be less than or equal to 10000 when completions are above 100000 and used with backoff limit per index": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					Completions:          pointer.Int32(100_001),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					MaxFailedIndexes:     pointer.Int32(50_000),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.maxFailedIndexes:must be greater than or equal to 0": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					BackoffLimitPerIndex: pointer.Int32(1),
+					MaxFailedIndexes:     pointer.Int32(-1),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
+		"spec.backoffLimitPerIndex: Required value: when maxFailedIndexes is specified": {
+			job: batch.Job{
+				ObjectMeta: validJobObjectMeta,
+				Spec: batch.JobSpec{
+					MaxFailedIndexes: pointer.Int32(1),
+					CompletionMode:   completionModePtr(batch.IndexedCompletion),
+					Selector:         validGeneratedSelector,
+					Template:         validPodTemplateSpecForGenerated,
+				},
+			},
+			opts: JobValidationOptions{RequirePrefixedLabels: true},
+		},
 		"spec.completions:must be greater than or equal to 0": {
 			job: batch.Job{
 				ObjectMeta: metav1.ObjectMeta{
@@ -1243,12 +1413,117 @@ func TestValidateJobUpdate(t *testing.T) {
 				Field: "spec.podFailurePolicy",
 			},
 		},
+		"set backoff limit per index": {
+			old: batch.Job{
+				ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
+				Spec: batch.JobSpec{
+					Selector:       validGeneratedSelector,
+					Template:       validPodTemplateSpecForGeneratedRestartPolicyNever,
+					Completions:    pointer.Int32(3),
+					CompletionMode: completionModePtr(batch.IndexedCompletion),
+				},
+			},
+			update: func(job *batch.Job) {
+				job.Spec.BackoffLimitPerIndex = pointer.Int32(1)
+			},
+			err: &field.Error{
+				Type:  field.ErrorTypeInvalid,
+				Field: "spec.backoffLimitPerIndex",
+			},
+		},
+		"unset backoff limit per index": {
+			old: batch.Job{
+				ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
+				Spec: batch.JobSpec{
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGeneratedRestartPolicyNever,
+					Completions:          pointer.Int32(3),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					BackoffLimitPerIndex: pointer.Int32(1),
+				},
+			},
+			update: func(job *batch.Job) {
+				job.Spec.BackoffLimitPerIndex = nil
+			},
+			err: &field.Error{
+				Type:  field.ErrorTypeInvalid,
+				Field: "spec.backoffLimitPerIndex",
+			},
+		},
+		"update backoff limit per index": {
+			old: batch.Job{
+				ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
+				Spec: batch.JobSpec{
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGeneratedRestartPolicyNever,
+					Completions:          pointer.Int32(3),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					BackoffLimitPerIndex: pointer.Int32(1),
+				},
+			},
+			update: func(job *batch.Job) {
+				job.Spec.BackoffLimitPerIndex = pointer.Int32(2)
+			},
+			err: &field.Error{
+				Type:  field.ErrorTypeInvalid,
+				Field: "spec.backoffLimitPerIndex",
+			},
+		},
+		"set max failed indexes": {
+			old: batch.Job{
+				ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
+				Spec: batch.JobSpec{
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGeneratedRestartPolicyNever,
+					Completions:          pointer.Int32(3),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					BackoffLimitPerIndex: pointer.Int32(1),
+				},
+			},
+			update: func(job *batch.Job) {
+				job.Spec.MaxFailedIndexes = pointer.Int32(1)
+			},
+		},
+		"unset max failed indexes": {
+			old: batch.Job{
+				ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
+				Spec: batch.JobSpec{
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGeneratedRestartPolicyNever,
+					Completions:          pointer.Int32(3),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					MaxFailedIndexes:     pointer.Int32(1),
+				},
+			},
+			update: func(job *batch.Job) {
+				job.Spec.MaxFailedIndexes = nil
+			},
+		},
+		"update max failed indexes": {
+			old: batch.Job{
+				ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
+				Spec: batch.JobSpec{
+					Selector:             validGeneratedSelector,
+					Template:             validPodTemplateSpecForGeneratedRestartPolicyNever,
+					Completions:          pointer.Int32(3),
+					CompletionMode:       completionModePtr(batch.IndexedCompletion),
+					BackoffLimitPerIndex: pointer.Int32(1),
+					MaxFailedIndexes:     pointer.Int32(1),
+				},
+			},
+			update: func(job *batch.Job) {
+				job.Spec.MaxFailedIndexes = pointer.Int32(2)
+			},
+		},
 		"immutable pod template": {
 			old: batch.Job{
 				ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
 				Spec: batch.JobSpec{
-					Selector: validGeneratedSelector,
-					Template: validPodTemplateSpecForGenerated,
+					Selector:       validGeneratedSelector,
+					Template:       validPodTemplateSpecForGenerated,
+					Completions:    pointer.Int32(3),
+					CompletionMode: completionModePtr(batch.IndexedCompletion),
 				},
 			},
 			update: func(job *batch.Job) {