diff --git a/hack/jenkins/job-configs/global.yaml b/hack/jenkins/job-configs/global.yaml index 9fc4f226fb9..a778e3e17b0 100644 --- a/hack/jenkins/job-configs/global.yaml +++ b/hack/jenkins/job-configs/global.yaml @@ -127,6 +127,10 @@ name: global emails: '$DEFAULT_RECIPIENTS' cron-string: 'H/30 * * * *' + # How long to wait after sending TERM to send KILL (minutes) + kill-timeout: 15 + # Just to be safe, use the Jenkins timeout after a long time. + jenkins-timeout: 600 branch: 'master' job-env: '' runner: curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh" | bash - diff --git a/hack/jenkins/job-configs/kubernetes-build.yaml b/hack/jenkins/job-configs/kubernetes-build.yaml index 31b94301af4..67e88789afe 100644 --- a/hack/jenkins/job-configs/kubernetes-build.yaml +++ b/hack/jenkins/job-configs/kubernetes-build.yaml @@ -6,7 +6,7 @@ numToKeep: 200 builders: - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' - - shell: './hack/jenkins/build.sh' + - shell: 'timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh' properties: - mail-watcher publishers: @@ -35,8 +35,8 @@ cron: 'H/2 * * * *' wrappers: - timeout: - timeout: 30 - abort: true + timeout: '{jenkins-timeout}' + fail: true - timestamps - project: diff --git a/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml b/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml index 6f4297efafd..a0a8b82232c 100644 --- a/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml +++ b/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml @@ -30,7 +30,7 @@ export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h" {post-env} export KUBE_GCE_NETWORK="e2e-enormous-cluster" - {runner} + timeout -k {kill-timeout}m 480m {runner} properties: - mail-watcher publishers: @@ -45,7 +45,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 480 + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup diff --git a/hack/jenkins/job-configs/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-e2e.yaml index 5ea3705fa8f..1b796c48941 100644 --- a/hack/jenkins/job-configs/kubernetes-e2e.yaml +++ b/hack/jenkins/job-configs/kubernetes-e2e.yaml @@ -21,14 +21,14 @@ {provider-env} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m {timeout}m {runner} properties: - mail-watcher wrappers: - ansicolor: colormap: xterm - timeout: - timeout: '{timeout}' + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup diff --git a/hack/jenkins/job-configs/kubernetes-kubemark.yaml b/hack/jenkins/job-configs/kubernetes-kubemark.yaml index 27e946e92e3..2560d4dcf24 100644 --- a/hack/jenkins/job-configs/kubernetes-kubemark.yaml +++ b/hack/jenkins/job-configs/kubernetes-kubemark.yaml @@ -8,7 +8,7 @@ {provider-env} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m {timeout}m {runner} properties: - mail-watcher publishers: @@ -27,7 +27,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: '{timeout}' + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup diff --git a/hack/jenkins/job-configs/kubernetes-soak.yaml b/hack/jenkins/job-configs/kubernetes-soak.yaml index 57c6b492f14..46669a98011 100644 --- a/hack/jenkins/job-configs/kubernetes-soak.yaml +++ b/hack/jenkins/job-configs/kubernetes-soak.yaml @@ -9,7 +9,7 @@ {soak-deploy} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 90m {runner} properties: - build-blocker: use-build-blocker: true @@ -24,7 +24,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 90 + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup @@ -41,7 +41,7 @@ {soak-continuous} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 360m {runner} properties: - build-blocker: use-build-blocker: true @@ -60,7 +60,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 360 + timeout: '{jenkins-timeout}' fail: true - timestamps diff --git a/hack/jenkins/job-configs/kubernetes-test-go.yaml b/hack/jenkins/job-configs/kubernetes-test-go.yaml index b9c2adc6e5c..0b425adf4d1 100644 --- a/hack/jenkins/job-configs/kubernetes-test-go.yaml +++ b/hack/jenkins/job-configs/kubernetes-test-go.yaml @@ -8,7 +8,7 @@ node: unittest builders: - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' - - shell: './hack/jenkins/gotest-dockerized.sh' + - shell: 'timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh' publishers: - claim-build - gcs-uploader @@ -47,7 +47,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: '{timeout}' + timeout: '{jenkins-timeout}' fail: true - timestamps - raw: diff --git a/hack/jenkins/job-configs/kubernetes-upgrades.yaml b/hack/jenkins/job-configs/kubernetes-upgrades.yaml index dec18d4f49c..61a99a724e1 100644 --- a/hack/jenkins/job-configs/kubernetes-upgrades.yaml +++ b/hack/jenkins/job-configs/kubernetes-upgrades.yaml @@ -70,7 +70,7 @@ # per-step variables, such as whether to run tests {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 60m {runner} properties: - mail-watcher publishers: @@ -84,7 +84,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 60 + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup @@ -105,7 +105,7 @@ # per-step variables, such as whether to run tests {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 300m {runner} properties: - mail-watcher publishers: @@ -119,8 +119,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 300 - abort: true + timeout: '{jenkins-timeout}' fail: true - timestamps # Don't clean the workspace; we want to keep configs intact across steps in the multijob