Use timeout command instead of Jenkins for e2e tests.

hack/jenkins/job-configs/global.yaml

@@ -127,6 +127,10 @@
     name: global
     emails: '$DEFAULT_RECIPIENTS'
     cron-string: 'H/30 * * * *'
+    # How long to wait after sending TERM to send KILL (minutes)
+    kill-timeout: 15
+    # Just to be safe, use the Jenkins timeout after a long time.
+    jenkins-timeout: 600
     branch: 'master'
     job-env: ''
     runner: curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh" | bash -

hack/jenkins/job-configs/kubernetes-build.yaml

@@ -6,7 +6,7 @@
         numToKeep: 200
     builders:
         - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
-        - shell: './hack/jenkins/build.sh'
+        - shell: 'timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh'
     properties:
         - mail-watcher
     publishers:
@@ -35,8 +35,8 @@
             cron: 'H/2 * * * *'
     wrappers:
         - timeout:
-            timeout: 30
+            timeout: '{jenkins-timeout}'
-            abort: true
+            fail: true
         - timestamps
 
 - project:

hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml

@@ -30,7 +30,7 @@
             export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h"
             {post-env}
             export KUBE_GCE_NETWORK="e2e-enormous-cluster"
-            {runner}
+            timeout -k {kill-timeout}m 480m {runner}
     properties:
         - mail-watcher
     publishers:
@@ -45,7 +45,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 480
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup

hack/jenkins/job-configs/kubernetes-e2e.yaml

@@ -21,14 +21,14 @@
             {provider-env}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m {timeout}m {runner}
     properties:
         - mail-watcher
     wrappers:
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: '{timeout}'
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup

hack/jenkins/job-configs/kubernetes-kubemark.yaml

@@ -8,7 +8,7 @@
             {provider-env}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m {timeout}m {runner}
     properties:
         - mail-watcher
     publishers:
@@ -27,7 +27,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: '{timeout}'
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup

hack/jenkins/job-configs/kubernetes-soak.yaml

@@ -9,7 +9,7 @@
             {soak-deploy}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 90m {runner}
     properties:
         - build-blocker:
             use-build-blocker: true
@@ -24,7 +24,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 90
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup
@@ -41,7 +41,7 @@
             {soak-continuous}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 360m {runner}
     properties:
         - build-blocker:
             use-build-blocker: true
@@ -60,7 +60,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 360
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
 

hack/jenkins/job-configs/kubernetes-test-go.yaml

@@ -8,7 +8,7 @@
     node: unittest
     builders:
         - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
-        - shell: './hack/jenkins/gotest-dockerized.sh'
+        - shell: 'timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh'
     publishers:
         - claim-build
         - gcs-uploader
@@ -47,7 +47,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: '{timeout}'
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - raw:

hack/jenkins/job-configs/kubernetes-upgrades.yaml

@@ -70,7 +70,7 @@
             # per-step variables, such as whether to run tests
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 60m {runner}
     properties:
         - mail-watcher
     publishers:
@@ -84,7 +84,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 60
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup
@@ -105,7 +105,7 @@
             # per-step variables, such as whether to run tests
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 300m {runner}
     properties:
         - mail-watcher
     publishers:
@@ -119,8 +119,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 300
+            timeout: '{jenkins-timeout}'
-            abort: true
             fail: true
         - timestamps
         # Don't clean the workspace; we want to keep configs intact across steps in the multijob