a46bab6930
* Fix a job quota related deadlock
In case ResourceQuota is used and sets a max # of jobs, a CronJob may get
trapped in a deadlock:
1. Job quota for a namespace is reached.
2. CronJob controller can't create a new job, because quota is
reached.
3. Cleanup of jobs owned by a cronjob doesn't happen, because a
control loop iteration is finished because of an error to create a
job.
To fix this we stop early quitting from a control loop iteration when
cronjob reconciliation failed and always let old jobs to be cleaned up.
* Dont reorder imports
* Don't stop requeuing on reconciliation error
Previous code only logged the reconciliation error inside jm.sync() and
didn't return the reconciliation error to it's invoker
processNextWorkItem().
Adding a copy-paste back to avoid this issue.
* Remove copy-pasted cleanupFinishedJobs()
Now we always call jm.cleanupFinishedJobs() first and then
jm.syncCronJob().
We also extract cronJobCopy and updateStatus outside jm.syncCronJob
function and pass pointers to them in both jm.syncCronJob and
jm.cleanupFinishedJobs to make delayed updates handling more explicit
and not dependent on the order in which cleanupFinishedJobs and
syncCronJob are invoked.
* Return updateStatus bool instead of changing the reference
* Explicitly ignore err in tests to fix linter
1815 lines
66 KiB
Go
1815 lines
66 KiB
Go
/*
|
|
Copyright 2020 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package cronjob
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/informers"
|
|
"k8s.io/client-go/kubernetes/fake"
|
|
"k8s.io/client-go/tools/record"
|
|
"k8s.io/client-go/util/workqueue"
|
|
"k8s.io/klog/v2/ktesting"
|
|
"k8s.io/utils/pointer"
|
|
|
|
_ "k8s.io/kubernetes/pkg/apis/batch/install"
|
|
_ "k8s.io/kubernetes/pkg/apis/core/install"
|
|
"k8s.io/kubernetes/pkg/controller"
|
|
)
|
|
|
|
var (
|
|
shortDead int64 = 10
|
|
mediumDead int64 = 2 * 60 * 60
|
|
longDead int64 = 1000000
|
|
noDead int64 = -12345
|
|
|
|
errorSchedule = "obvious error schedule"
|
|
// schedule is hourly on the hour
|
|
onTheHour = "0 * * * ?"
|
|
everyHour = "@every 1h"
|
|
|
|
errorTimeZone = "bad timezone"
|
|
newYork = "America/New_York"
|
|
)
|
|
|
|
// returns a cronJob with some fields filled in.
|
|
func cronJob() batchv1.CronJob {
|
|
return batchv1.CronJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "mycronjob",
|
|
Namespace: "snazzycats",
|
|
UID: types.UID("1a2b3c"),
|
|
CreationTimestamp: metav1.Time{Time: justBeforeTheHour()},
|
|
},
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: "* * * * ?",
|
|
ConcurrencyPolicy: "Allow",
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "b"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func jobSpec() batchv1.JobSpec {
|
|
one := int32(1)
|
|
return batchv1.JobSpec{
|
|
Parallelism: &one,
|
|
Completions: &one,
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"foo": "bar",
|
|
},
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{Image: "foo/bar"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func justASecondBeforeTheHour() time.Time {
|
|
T1, err := time.Parse(time.RFC3339, "2016-05-19T09:59:59Z")
|
|
if err != nil {
|
|
panic("test setup error")
|
|
}
|
|
return T1
|
|
}
|
|
|
|
func justAfterThePriorHour() time.Time {
|
|
T1, err := time.Parse(time.RFC3339, "2016-05-19T09:01:00Z")
|
|
if err != nil {
|
|
panic("test setup error")
|
|
}
|
|
return T1
|
|
}
|
|
|
|
func justBeforeThePriorHour() time.Time {
|
|
T1, err := time.Parse(time.RFC3339, "2016-05-19T08:59:00Z")
|
|
if err != nil {
|
|
panic("test setup error")
|
|
}
|
|
return T1
|
|
}
|
|
|
|
func justAfterTheHour() *time.Time {
|
|
T1, err := time.Parse(time.RFC3339, "2016-05-19T10:01:00Z")
|
|
if err != nil {
|
|
panic("test setup error")
|
|
}
|
|
return &T1
|
|
}
|
|
|
|
func justAfterTheHourInZone(tz string) time.Time {
|
|
location, err := time.LoadLocation(tz)
|
|
if err != nil {
|
|
panic("tz error: " + err.Error())
|
|
}
|
|
|
|
T1, err := time.ParseInLocation(time.RFC3339, "2016-05-19T10:01:00Z", location)
|
|
if err != nil {
|
|
panic("test setup error: " + err.Error())
|
|
}
|
|
return T1
|
|
}
|
|
|
|
func justBeforeTheHour() time.Time {
|
|
T1, err := time.Parse(time.RFC3339, "2016-05-19T09:59:00Z")
|
|
if err != nil {
|
|
panic("test setup error")
|
|
}
|
|
return T1
|
|
}
|
|
|
|
func justBeforeTheNextHour() time.Time {
|
|
T1, err := time.Parse(time.RFC3339, "2016-05-19T10:59:00Z")
|
|
if err != nil {
|
|
panic("test setup error")
|
|
}
|
|
return T1
|
|
}
|
|
|
|
func weekAfterTheHour() time.Time {
|
|
T1, err := time.Parse(time.RFC3339, "2016-05-26T10:00:00Z")
|
|
if err != nil {
|
|
panic("test setup error")
|
|
}
|
|
return T1
|
|
}
|
|
|
|
func TestControllerV2SyncCronJob(t *testing.T) {
|
|
// Check expectations on deadline parameters
|
|
if shortDead/60/60 >= 1 {
|
|
t.Errorf("shortDead should be less than one hour")
|
|
}
|
|
|
|
if mediumDead/60/60 < 1 || mediumDead/60/60 >= 24 {
|
|
t.Errorf("mediumDead should be between one hour and one day")
|
|
}
|
|
|
|
if longDead/60/60/24 < 10 {
|
|
t.Errorf("longDead should be at least ten days")
|
|
}
|
|
|
|
testCases := map[string]struct {
|
|
// cj spec
|
|
concurrencyPolicy batchv1.ConcurrencyPolicy
|
|
suspend bool
|
|
schedule string
|
|
timeZone *string
|
|
deadline int64
|
|
|
|
// cj status
|
|
ranPreviously bool
|
|
stillActive bool
|
|
|
|
// environment
|
|
cronjobCreationTime time.Time
|
|
jobCreationTime time.Time
|
|
lastScheduleTime time.Time
|
|
now time.Time
|
|
jobCreateError error
|
|
jobGetErr error
|
|
|
|
// expectations
|
|
expectCreate bool
|
|
expectDelete bool
|
|
expectActive int
|
|
expectedWarnings int
|
|
expectErr bool
|
|
expectRequeueAfter bool
|
|
expectedRequeueDuration time.Duration
|
|
expectUpdateStatus bool
|
|
jobStillNotFoundInLister bool
|
|
jobPresentInCJActiveStatus bool
|
|
}{
|
|
"never ran, not valid schedule, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: errorSchedule,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectedWarnings: 1,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, not valid schedule, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: errorSchedule,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectedWarnings: 1,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, not valid schedule, R": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: errorSchedule,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectedWarnings: 1,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, not valid time zone": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
timeZone: &errorTimeZone,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectedWarnings: 1,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, not time, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true},
|
|
"never ran, not time, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, not time, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, not time in zone": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
timeZone: &newYork,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time in zone, but time zone disabled": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
timeZone: &newYork,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justAfterTheHourInZone(newYork),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time in zone": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
timeZone: &newYork,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justAfterTheHourInZone(newYork),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time in zone, but TZ is also set in schedule": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: "TZ=UTC " + onTheHour,
|
|
timeZone: &newYork,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justAfterTheHourInZone(newYork),
|
|
expectCreate: true,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time, suspended": {
|
|
concurrencyPolicy: "Allow",
|
|
suspend: true,
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time, past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: shortDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justAfterTheHour().Add(time.Minute * time.Duration(shortDead+1)),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Minute*time.Duration(shortDead+1) + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"never ran, is time, not past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
"prev ran but done, not time, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, not time, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, not time, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, is time, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, is time, create job failed, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
jobCreateError: errors.NewAlreadyExists(schema.GroupResource{Resource: "job", Group: "batch"}, ""),
|
|
expectErr: true,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, is time, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, is time, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, is time, suspended": {
|
|
concurrencyPolicy: "Allow",
|
|
suspend: true,
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, is time, past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, is time, not past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
"still active, not time, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, not time, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, not time, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, is time, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 2,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, is time, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, is time, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectDelete: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, is time, get job failed, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
jobGetErr: errors.NewBadRequest("request is invalid"),
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, is time, suspended": {
|
|
concurrencyPolicy: "Allow",
|
|
suspend: true,
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectActive: 1,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, is time, past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"still active, is time, not past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 2,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
// Controller should fail to schedule these, as there are too many missed starting times
|
|
// and either no deadline or a too long deadline.
|
|
"prev ran but done, long overdue, not past deadline, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, not past deadline, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, not past deadline, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, no deadline, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, no deadline, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, no deadline, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
"prev ran but done, long overdue, past medium deadline, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: mediumDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, past short deadline, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
"prev ran but done, long overdue, past medium deadline, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: mediumDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, past short deadline, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
"prev ran but done, long overdue, past medium deadline, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: mediumDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"prev ran but done, long overdue, past short deadline, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
// Tests for time skews
|
|
// the controller sees job is created, takes no actions
|
|
"this ran but done, time drifted back, F": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
jobCreationTime: *justAfterTheHour(),
|
|
now: justBeforeTheHour(),
|
|
jobCreateError: errors.NewAlreadyExists(schema.GroupResource{Resource: "jobs", Group: "batch"}, ""),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
},
|
|
|
|
// Tests for slow job lister
|
|
"this started but went missing, not past deadline, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: topOfTheHour().Add(time.Millisecond * 100),
|
|
now: justAfterTheHour().Add(time.Millisecond * 100),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
|
|
jobStillNotFoundInLister: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"this started but went missing, not past deadline, f": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: topOfTheHour().Add(time.Millisecond * 100),
|
|
now: justAfterTheHour().Add(time.Millisecond * 100),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
|
|
jobStillNotFoundInLister: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"this started but went missing, not past deadline, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: topOfTheHour().Add(time.Millisecond * 100),
|
|
now: justAfterTheHour().Add(time.Millisecond * 100),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
|
|
jobStillNotFoundInLister: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
|
|
// Tests for slow cronjob list
|
|
"this started but is not present in cronjob active list, not past deadline, A": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: topOfTheHour().Add(time.Millisecond * 100),
|
|
now: justAfterTheHour().Add(time.Millisecond * 100),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
|
|
},
|
|
"this started but is not present in cronjob active list, not past deadline, f": {
|
|
concurrencyPolicy: "Forbid",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: topOfTheHour().Add(time.Millisecond * 100),
|
|
now: justAfterTheHour().Add(time.Millisecond * 100),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
|
|
},
|
|
"this started but is not present in cronjob active list, not past deadline, R": {
|
|
concurrencyPolicy: "Replace",
|
|
schedule: onTheHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: topOfTheHour().Add(time.Millisecond * 100),
|
|
now: justAfterTheHour().Add(time.Millisecond * 100),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
|
|
},
|
|
|
|
// Tests for @every-style schedule
|
|
"with @every schedule, never ran, not time": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: noDead,
|
|
cronjobCreationTime: justBeforeTheHour(),
|
|
jobCreationTime: justBeforeTheHour(),
|
|
now: *topOfTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, never ran, is time": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: noDead,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeThePriorHour(),
|
|
now: justBeforeTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectUpdateStatus: true,
|
|
},
|
|
"with @every schedule, never ran, is time, past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: shortDead,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeThePriorHour(),
|
|
now: justBeforeTheHour().Add(time.Second * time.Duration(shortDead+1)),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, never ran, is time, not past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: longDead,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeThePriorHour(),
|
|
now: justBeforeTheHour().Add(time.Second * time.Duration(shortDead-1)),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - time.Second*time.Duration(shortDead-1) + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, prev ran but done, not time": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeThePriorHour(),
|
|
lastScheduleTime: justBeforeTheHour(),
|
|
now: *topOfTheHour(),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, prev ran but done, is time": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeThePriorHour(),
|
|
lastScheduleTime: justBeforeTheHour(),
|
|
now: topOfTheHour().Add(1 * time.Hour),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, prev ran but done, is time, past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeThePriorHour(),
|
|
lastScheduleTime: justBeforeTheHour(),
|
|
now: justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead+1)),
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
// This test will fail: the logic around StartingDeadlineSecond in getNextScheduleTime messes up
|
|
// the time that calculating schedule.Next(earliestTime) is based on. While this works perfectly
|
|
// well for classic cron scheduled, with @every X, schedule.Next(earliestTime) just returns the time
|
|
// offset by X relative to the earliestTime.
|
|
// "with @every schedule, prev ran but done, is time, not past deadline": {
|
|
// concurrencyPolicy: "Allow",
|
|
// schedule: everyHour,
|
|
// deadline: shortDead,
|
|
// ranPreviously: true,
|
|
// cronjobCreationTime: justBeforeThePriorHour(),
|
|
// jobCreationTime: justBeforeThePriorHour(),
|
|
// lastScheduleTime: justBeforeTheHour(),
|
|
// now: justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead-1)),
|
|
// expectCreate: true,
|
|
// expectActive: 1,
|
|
// expectRequeueAfter: true,
|
|
// expectedRequeueDuration: 1*time.Hour - time.Second*time.Duration(shortDead-1) + nextScheduleDelta,
|
|
// expectUpdateStatus: true,
|
|
// jobPresentInCJActiveStatus: true,
|
|
// },
|
|
"with @every schedule, still active, not time": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeTheHour(),
|
|
lastScheduleTime: justBeforeTheHour(),
|
|
now: *topOfTheHour(),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 1*time.Minute + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, still active, is time": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeThePriorHour(),
|
|
lastScheduleTime: justBeforeThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 2,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - 2*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, still active, is time, past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeTheHour(),
|
|
lastScheduleTime: justBeforeTheHour(),
|
|
now: justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead+1)),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, still active, is time, not past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: longDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
cronjobCreationTime: justBeforeThePriorHour(),
|
|
jobCreationTime: justBeforeTheHour(),
|
|
lastScheduleTime: justBeforeTheHour(),
|
|
now: justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead-1)),
|
|
expectCreate: true,
|
|
expectActive: 2,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - time.Second*time.Duration(shortDead-1) + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, prev ran but done, long overdue, no deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
cronjobCreationTime: justAfterThePriorHour(),
|
|
lastScheduleTime: *justAfterTheHour(),
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour(),
|
|
expectCreate: true,
|
|
expectActive: 1,
|
|
expectedWarnings: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"with @every schedule, prev ran but done, long overdue, past deadline": {
|
|
concurrencyPolicy: "Allow",
|
|
schedule: everyHour,
|
|
deadline: shortDead,
|
|
ranPreviously: true,
|
|
cronjobCreationTime: justAfterThePriorHour(),
|
|
lastScheduleTime: *justAfterTheHour(),
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: weekAfterTheHour().Add(1 * time.Minute).Add(time.Second * time.Duration(shortDead+1)),
|
|
expectActive: 1,
|
|
expectRequeueAfter: true,
|
|
expectedRequeueDuration: 1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
|
|
expectUpdateStatus: true,
|
|
jobPresentInCJActiveStatus: true,
|
|
},
|
|
"do nothing if the namespace is terminating": {
|
|
jobCreateError: &errors.StatusError{ErrStatus: metav1.Status{Details: &metav1.StatusDetails{Causes: []metav1.StatusCause{
|
|
{
|
|
Type: v1.NamespaceTerminatingCause,
|
|
Message: fmt.Sprintf("namespace %s is being terminated", metav1.NamespaceDefault),
|
|
Field: "metadata.namespace",
|
|
}}}}},
|
|
concurrencyPolicy: "Allow",
|
|
schedule: onTheHour,
|
|
deadline: noDead,
|
|
ranPreviously: true,
|
|
stillActive: true,
|
|
jobCreationTime: justAfterThePriorHour(),
|
|
now: *justAfterTheHour(),
|
|
expectActive: 0,
|
|
expectRequeueAfter: false,
|
|
expectUpdateStatus: false,
|
|
expectErr: true,
|
|
jobPresentInCJActiveStatus: false,
|
|
},
|
|
}
|
|
for name, tc := range testCases {
|
|
name := name
|
|
tc := tc
|
|
|
|
t.Run(name, func(t *testing.T) {
|
|
cj := cronJob()
|
|
cj.Spec.ConcurrencyPolicy = tc.concurrencyPolicy
|
|
cj.Spec.Suspend = &tc.suspend
|
|
cj.Spec.Schedule = tc.schedule
|
|
cj.Spec.TimeZone = tc.timeZone
|
|
if tc.deadline != noDead {
|
|
cj.Spec.StartingDeadlineSeconds = &tc.deadline
|
|
}
|
|
|
|
var (
|
|
job *batchv1.Job
|
|
err error
|
|
)
|
|
js := []*batchv1.Job{}
|
|
realCJ := cj.DeepCopy()
|
|
if tc.ranPreviously {
|
|
cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: justBeforeThePriorHour()}
|
|
if !tc.cronjobCreationTime.IsZero() {
|
|
cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: tc.cronjobCreationTime}
|
|
}
|
|
cj.Status.LastScheduleTime = &metav1.Time{Time: justAfterThePriorHour()}
|
|
if !tc.lastScheduleTime.IsZero() {
|
|
cj.Status.LastScheduleTime = &metav1.Time{Time: tc.lastScheduleTime}
|
|
}
|
|
job, err = getJobFromTemplate2(&cj, tc.jobCreationTime)
|
|
if err != nil {
|
|
t.Fatalf("%s: unexpected error creating a job from template: %v", name, err)
|
|
}
|
|
job.UID = "1234"
|
|
job.Namespace = cj.Namespace
|
|
if tc.stillActive {
|
|
ref, err := getRef(job)
|
|
if err != nil {
|
|
t.Fatalf("%s: unexpected error getting the job object reference: %v", name, err)
|
|
}
|
|
if tc.jobPresentInCJActiveStatus {
|
|
cj.Status.Active = []v1.ObjectReference{*ref}
|
|
}
|
|
realCJ.Status.Active = []v1.ObjectReference{*ref}
|
|
if !tc.jobStillNotFoundInLister {
|
|
js = append(js, job)
|
|
}
|
|
} else {
|
|
job.Status.CompletionTime = &metav1.Time{Time: job.ObjectMeta.CreationTimestamp.Add(time.Second * 10)}
|
|
job.Status.Conditions = append(job.Status.Conditions, batchv1.JobCondition{
|
|
Type: batchv1.JobComplete,
|
|
Status: v1.ConditionTrue,
|
|
})
|
|
if !tc.jobStillNotFoundInLister {
|
|
js = append(js, job)
|
|
}
|
|
}
|
|
} else {
|
|
cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: justBeforeTheHour()}
|
|
if !tc.cronjobCreationTime.IsZero() {
|
|
cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: tc.cronjobCreationTime}
|
|
}
|
|
if tc.stillActive {
|
|
t.Errorf("%s: test setup error: this case makes no sense", name)
|
|
}
|
|
}
|
|
|
|
jc := &fakeJobControl{Job: job, CreateErr: tc.jobCreateError, Err: tc.jobGetErr}
|
|
cjc := &fakeCJControl{CronJob: realCJ}
|
|
recorder := record.NewFakeRecorder(10)
|
|
|
|
jm := ControllerV2{
|
|
jobControl: jc,
|
|
cronJobControl: cjc,
|
|
recorder: recorder,
|
|
now: func() time.Time {
|
|
return tc.now
|
|
},
|
|
}
|
|
cjCopy := cj.DeepCopy()
|
|
requeueAfter, updateStatus, err := jm.syncCronJob(context.TODO(), cjCopy, js)
|
|
if tc.expectErr && err == nil {
|
|
t.Errorf("%s: expected error got none with requeueAfter time: %#v", name, requeueAfter)
|
|
}
|
|
if tc.expectRequeueAfter {
|
|
if !reflect.DeepEqual(requeueAfter, &tc.expectedRequeueDuration) {
|
|
t.Errorf("%s: expected requeueAfter: %+v, got requeueAfter time: %+v", name, tc.expectedRequeueDuration, requeueAfter)
|
|
}
|
|
}
|
|
if updateStatus != tc.expectUpdateStatus {
|
|
t.Errorf("%s: expected updateStatus: %t, actually: %t", name, tc.expectUpdateStatus, updateStatus)
|
|
}
|
|
expectedCreates := 0
|
|
if tc.expectCreate {
|
|
expectedCreates = 1
|
|
}
|
|
if tc.ranPreviously && !tc.stillActive {
|
|
completionTime := tc.jobCreationTime.Add(10 * time.Second)
|
|
if cjCopy.Status.LastSuccessfulTime == nil || !cjCopy.Status.LastSuccessfulTime.Time.Equal(completionTime) {
|
|
t.Errorf("cj.status.lastSuccessfulTime: %s expected, got %#v", completionTime, cj.Status.LastSuccessfulTime)
|
|
}
|
|
}
|
|
if len(jc.Jobs) != expectedCreates {
|
|
t.Errorf("%s: expected %d job started, actually %v", name, expectedCreates, len(jc.Jobs))
|
|
}
|
|
for i := range jc.Jobs {
|
|
job := &jc.Jobs[i]
|
|
controllerRef := metav1.GetControllerOf(job)
|
|
if controllerRef == nil {
|
|
t.Errorf("%s: expected job to have ControllerRef: %#v", name, job)
|
|
} else {
|
|
if got, want := controllerRef.APIVersion, "batch/v1"; got != want {
|
|
t.Errorf("%s: controllerRef.APIVersion = %q, want %q", name, got, want)
|
|
}
|
|
if got, want := controllerRef.Kind, "CronJob"; got != want {
|
|
t.Errorf("%s: controllerRef.Kind = %q, want %q", name, got, want)
|
|
}
|
|
if got, want := controllerRef.Name, cj.Name; got != want {
|
|
t.Errorf("%s: controllerRef.Name = %q, want %q", name, got, want)
|
|
}
|
|
if got, want := controllerRef.UID, cj.UID; got != want {
|
|
t.Errorf("%s: controllerRef.UID = %q, want %q", name, got, want)
|
|
}
|
|
if controllerRef.Controller == nil || *controllerRef.Controller != true {
|
|
t.Errorf("%s: controllerRef.Controller is not set to true", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
expectedDeletes := 0
|
|
if tc.expectDelete {
|
|
expectedDeletes = 1
|
|
}
|
|
if len(jc.DeleteJobName) != expectedDeletes {
|
|
t.Errorf("%s: expected %d job deleted, actually %v", name, expectedDeletes, len(jc.DeleteJobName))
|
|
}
|
|
|
|
// Status update happens once when ranging through job list, and another one if create jobs.
|
|
expectUpdates := 1
|
|
expectedEvents := 0
|
|
if tc.expectCreate {
|
|
expectedEvents++
|
|
expectUpdates++
|
|
}
|
|
if tc.expectDelete {
|
|
expectedEvents++
|
|
}
|
|
if name == "still active, is time, F" {
|
|
// this is the only test case where we would raise an event for not scheduling
|
|
expectedEvents++
|
|
}
|
|
expectedEvents += tc.expectedWarnings
|
|
|
|
if len(recorder.Events) != expectedEvents {
|
|
t.Errorf("%s: expected %d event, actually %v", name, expectedEvents, len(recorder.Events))
|
|
}
|
|
|
|
numWarnings := 0
|
|
for i := 1; i <= len(recorder.Events); i++ {
|
|
e := <-recorder.Events
|
|
if strings.HasPrefix(e, v1.EventTypeWarning) {
|
|
numWarnings++
|
|
}
|
|
}
|
|
if numWarnings != tc.expectedWarnings {
|
|
t.Errorf("%s: expected %d warnings, actually %v", name, tc.expectedWarnings, numWarnings)
|
|
}
|
|
|
|
if len(cjc.Updates) == expectUpdates && tc.expectActive != len(cjc.Updates[expectUpdates-1].Status.Active) {
|
|
t.Errorf("%s: expected Active size %d, got %d", name, tc.expectActive, len(cjc.Updates[expectUpdates-1].Status.Active))
|
|
}
|
|
|
|
if &cj == cjCopy {
|
|
t.Errorf("syncCronJob is not creating a copy of the original cronjob")
|
|
}
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
type fakeQueue struct {
|
|
workqueue.RateLimitingInterface
|
|
delay time.Duration
|
|
key interface{}
|
|
}
|
|
|
|
func (f *fakeQueue) AddAfter(key interface{}, delay time.Duration) {
|
|
f.delay = delay
|
|
f.key = key
|
|
}
|
|
|
|
// this test will take around 61 seconds to complete
|
|
func TestControllerV2UpdateCronJob(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
oldCronJob *batchv1.CronJob
|
|
newCronJob *batchv1.CronJob
|
|
expectedDelay time.Duration
|
|
}{
|
|
{
|
|
name: "spec.template changed",
|
|
oldCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "b"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
},
|
|
newCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "foo"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
},
|
|
expectedDelay: 0 * time.Second,
|
|
},
|
|
{
|
|
name: "spec.schedule changed",
|
|
oldCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: "30 * * * *",
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "b"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{
|
|
LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
|
|
},
|
|
},
|
|
newCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: "*/1 * * * *",
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "foo"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{
|
|
LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
|
|
},
|
|
},
|
|
expectedDelay: 1*time.Second + nextScheduleDelta,
|
|
},
|
|
{
|
|
name: "spec.schedule with @every changed - cadence decrease",
|
|
oldCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: "@every 1m",
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "b"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{
|
|
LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
|
|
},
|
|
},
|
|
newCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: "@every 3m",
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "foo"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{
|
|
LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
|
|
},
|
|
},
|
|
expectedDelay: 2*time.Minute + 1*time.Second + nextScheduleDelta,
|
|
},
|
|
{
|
|
name: "spec.schedule with @every changed - cadence increase",
|
|
oldCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: "@every 3m",
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "b"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{
|
|
LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
|
|
},
|
|
},
|
|
newCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: "@every 1m",
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "foo"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{
|
|
LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
|
|
},
|
|
},
|
|
expectedDelay: 1*time.Second + nextScheduleDelta,
|
|
},
|
|
{
|
|
name: "spec.timeZone not changed",
|
|
oldCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
TimeZone: &newYork,
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "b"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
},
|
|
newCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
TimeZone: &newYork,
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "foo"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
},
|
|
expectedDelay: 0 * time.Second,
|
|
},
|
|
{
|
|
name: "spec.timeZone changed",
|
|
oldCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
TimeZone: &newYork,
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "b"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
},
|
|
newCronJob: &batchv1.CronJob{
|
|
Spec: batchv1.CronJobSpec{
|
|
TimeZone: nil,
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"a": "foo"},
|
|
Annotations: map[string]string{"x": "y"},
|
|
},
|
|
Spec: jobSpec(),
|
|
},
|
|
},
|
|
},
|
|
expectedDelay: 0 * time.Second,
|
|
},
|
|
|
|
// TODO: Add more test cases for updating scheduling.
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
logger, ctx := ktesting.NewTestContext(t)
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
kubeClient := fake.NewSimpleClientset()
|
|
sharedInformers := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc())
|
|
jm, err := NewControllerV2(ctx, sharedInformers.Batch().V1().Jobs(), sharedInformers.Batch().V1().CronJobs(), kubeClient)
|
|
if err != nil {
|
|
t.Errorf("unexpected error %v", err)
|
|
return
|
|
}
|
|
jm.now = justASecondBeforeTheHour
|
|
queue := &fakeQueue{RateLimitingInterface: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test-update-cronjob")}
|
|
jm.queue = queue
|
|
jm.jobControl = &fakeJobControl{}
|
|
jm.cronJobControl = &fakeCJControl{}
|
|
jm.recorder = record.NewFakeRecorder(10)
|
|
|
|
jm.updateCronJob(logger, tt.oldCronJob, tt.newCronJob)
|
|
if queue.delay.Seconds() != tt.expectedDelay.Seconds() {
|
|
t.Errorf("Expected delay %#v got %#v", tt.expectedDelay.Seconds(), queue.delay.Seconds())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestControllerV2GetJobsToBeReconciled(t *testing.T) {
|
|
trueRef := true
|
|
tests := []struct {
|
|
name string
|
|
cronJob *batchv1.CronJob
|
|
jobs []runtime.Object
|
|
expected []*batchv1.Job
|
|
}{
|
|
{
|
|
name: "test getting jobs in namespace without controller reference",
|
|
cronJob: &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"}},
|
|
jobs: []runtime.Object{
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "foo-ns"}},
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "foo-ns"}},
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo2", Namespace: "foo-ns"}},
|
|
},
|
|
expected: []*batchv1.Job{},
|
|
},
|
|
{
|
|
name: "test getting jobs in namespace with a controller reference",
|
|
cronJob: &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"}},
|
|
jobs: []runtime.Object{
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "foo-ns"}},
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "foo-ns",
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}}},
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo2", Namespace: "foo-ns"}},
|
|
},
|
|
expected: []*batchv1.Job{
|
|
{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "foo-ns",
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}}},
|
|
},
|
|
},
|
|
{
|
|
name: "test getting jobs in other namespaces",
|
|
cronJob: &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"}},
|
|
jobs: []runtime.Object{
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar-ns"}},
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "bar-ns"}},
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo2", Namespace: "bar-ns"}},
|
|
},
|
|
expected: []*batchv1.Job{},
|
|
},
|
|
{
|
|
name: "test getting jobs whose labels do not match job template",
|
|
cronJob: &batchv1.CronJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"},
|
|
Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"key": "value"}},
|
|
}},
|
|
},
|
|
jobs: []runtime.Object{
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "foo-ns",
|
|
Name: "foo-fooer-owner-ref",
|
|
Labels: map[string]string{"key": "different-value"},
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}},
|
|
},
|
|
&batchv1.Job{ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "foo-ns",
|
|
Name: "foo-other-owner-ref",
|
|
Labels: map[string]string{"key": "different-value"},
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "another-cronjob", Controller: &trueRef}}},
|
|
},
|
|
},
|
|
expected: []*batchv1.Job{{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "foo-ns",
|
|
Name: "foo-fooer-owner-ref",
|
|
Labels: map[string]string{"key": "different-value"},
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}},
|
|
}},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
_, ctx := ktesting.NewTestContext(t)
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
kubeClient := fake.NewSimpleClientset()
|
|
sharedInformers := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc())
|
|
for _, job := range tt.jobs {
|
|
sharedInformers.Batch().V1().Jobs().Informer().GetIndexer().Add(job)
|
|
}
|
|
jm, err := NewControllerV2(ctx, sharedInformers.Batch().V1().Jobs(), sharedInformers.Batch().V1().CronJobs(), kubeClient)
|
|
if err != nil {
|
|
t.Errorf("unexpected error %v", err)
|
|
return
|
|
}
|
|
|
|
actual, err := jm.getJobsToBeReconciled(tt.cronJob)
|
|
if err != nil {
|
|
t.Errorf("unexpected error %v", err)
|
|
return
|
|
}
|
|
if !reflect.DeepEqual(actual, tt.expected) {
|
|
t.Errorf("\nExpected %#v,\nbut got %#v", tt.expected, actual)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestControllerV2CleanupFinishedJobs(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
now time.Time
|
|
cronJob *batchv1.CronJob
|
|
finishedJobs []*batchv1.Job
|
|
jobCreateError error
|
|
expectedDeletedJobs []string
|
|
}{
|
|
{
|
|
name: "jobs are still deleted when a cronjob can't create jobs due to jobs quota being reached (avoiding a deadlock)",
|
|
now: *justAfterTheHour(),
|
|
cronJob: &batchv1.CronJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"},
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: onTheHour,
|
|
SuccessfulJobsHistoryLimit: pointer.Int32(1),
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"key": "value"}},
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{LastScheduleTime: &metav1.Time{Time: justAfterThePriorHour()}},
|
|
},
|
|
finishedJobs: []*batchv1.Job{
|
|
{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "foo-ns",
|
|
Name: "finished-job-started-hour-ago",
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: pointer.Bool(true)}},
|
|
},
|
|
Status: batchv1.JobStatus{StartTime: &metav1.Time{Time: justBeforeThePriorHour()}},
|
|
},
|
|
{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "foo-ns",
|
|
Name: "finished-job-started-minute-ago",
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: pointer.Bool(true)}},
|
|
},
|
|
Status: batchv1.JobStatus{StartTime: &metav1.Time{Time: justBeforeTheHour()}},
|
|
},
|
|
},
|
|
jobCreateError: errors.NewInternalError(fmt.Errorf("quota for # of jobs reached")),
|
|
expectedDeletedJobs: []string{"finished-job-started-hour-ago"},
|
|
},
|
|
{
|
|
name: "jobs are not deleted if history limit not reached",
|
|
now: justBeforeTheHour(),
|
|
cronJob: &batchv1.CronJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"},
|
|
Spec: batchv1.CronJobSpec{
|
|
Schedule: onTheHour,
|
|
SuccessfulJobsHistoryLimit: pointer.Int32(2),
|
|
JobTemplate: batchv1.JobTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"key": "value"}},
|
|
},
|
|
},
|
|
Status: batchv1.CronJobStatus{LastScheduleTime: &metav1.Time{Time: justAfterThePriorHour()}},
|
|
},
|
|
finishedJobs: []*batchv1.Job{
|
|
{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "foo-ns",
|
|
Name: "finished-job-started-hour-ago",
|
|
OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: pointer.Bool(true)}},
|
|
},
|
|
Status: batchv1.JobStatus{StartTime: &metav1.Time{Time: justBeforeThePriorHour()}},
|
|
},
|
|
},
|
|
jobCreateError: nil,
|
|
expectedDeletedJobs: []string{},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
_, ctx := ktesting.NewTestContext(t)
|
|
|
|
for _, job := range tt.finishedJobs {
|
|
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: v1.ConditionTrue}}
|
|
}
|
|
|
|
client := fake.NewSimpleClientset()
|
|
|
|
informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
|
|
_ = informerFactory.Batch().V1().CronJobs().Informer().GetIndexer().Add(tt.cronJob)
|
|
for _, job := range tt.finishedJobs {
|
|
_ = informerFactory.Batch().V1().Jobs().Informer().GetIndexer().Add(job)
|
|
}
|
|
|
|
jm, err := NewControllerV2(ctx, informerFactory.Batch().V1().Jobs(), informerFactory.Batch().V1().CronJobs(), client)
|
|
if err != nil {
|
|
t.Errorf("unexpected error %v", err)
|
|
return
|
|
}
|
|
jobControl := &fakeJobControl{CreateErr: tt.jobCreateError}
|
|
jm.jobControl = jobControl
|
|
jm.now = func() time.Time {
|
|
return tt.now
|
|
}
|
|
|
|
jm.enqueueController(tt.cronJob)
|
|
jm.processNextWorkItem(ctx)
|
|
|
|
if len(tt.expectedDeletedJobs) != len(jobControl.DeleteJobName) {
|
|
t.Fatalf("expected '%v' jobs to be deleted, instead deleted '%s'", tt.expectedDeletedJobs, jobControl.DeleteJobName)
|
|
}
|
|
sort.Strings(jobControl.DeleteJobName)
|
|
sort.Strings(tt.expectedDeletedJobs)
|
|
for i, deletedJob := range jobControl.DeleteJobName {
|
|
if deletedJob != tt.expectedDeletedJobs[i] {
|
|
t.Fatalf("expected '%v' jobs to be deleted, instead deleted '%s'", tt.expectedDeletedJobs, jobControl.DeleteJobName)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|