github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove [Skipped] as a label for tests.

Browse Source
This commit is contained in:
Isaac Hollander McCreery
2016-02-09 16:06:46 -08:00
parent b8aa23f0fa
commit c09da9e03c
4 changed files with 14 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/devel/e2e-tests.md
View File

@@ -185,7 +185,6 @@ We are working on implementing clearer partitioning of our e2e tests to make run
- `[Serial]`: If a test cannot be run in parallel with other tests (e.g. it takes too many resources or restarts nodes), it is labeled `[Serial]`, and should be run in serial as part of a separate suite.
- `[Disruptive]`: If a test restarts components that might cause other tests to fail or break the cluster completely, it is labeled `[Disruptive]`. Any `[Disruptive]` test is also assumed to qualify for the `[Serial]` label, but need not be labeled as both. These tests are not run against soak clusters to avoid restarting components.
- `[Flaky]`: If a test is found to be flaky and we have decided that it's too hard to fix in the short term (e.g. it's going to take a full engineer-week), it receives the `[Flaky]` label until it is fixed. The `[Flaky]` label should be used very sparingly, and should be accompanied with a reference to the issue for de-flaking the test, because while a test remains labeled `[Flaky]`, it is not monitored closely in CI. `[Flaky]` tests are by default not run, unless a `focus` or `skip` argument is explicitly given.
- `[Skipped]`: `[Skipped]` is a legacy label that we're phasing out. If a test is marked `[Skipped]`, there should be an issue open to label it properly. `[Skipped]` tests are by default not run, unless a `focus` or `skip` argument is explicitly given.
- `[Feature:.+]`: If a test has non-default requirements to run or targets some non-core functionality, and thus should not be run as part of the standard suite, it receives a `[Feature:.+]` label, e.g. `[Feature:Performance]` or `[Feature:Ingress]`. `[Feature:.+]` tests are not run in our core suites, instead running in custom suites. If a feature is experimental or alpha and is not enabled by default due to being incomplete or potentially subject to breaking changes, it does *not* block the merge-queue, and thus should run in some separate test suites owned by the feature owner(s) (see #continuous_integration below).
### Conformance tests

2
hack/conformance-test.sh
View File

@@ -84,4 +84,4 @@ echo "Conformance test checking conformance with Kubernetes version 1.0"
declare -x KUBERNETES_CONFORMANCE_TEST="y"
declare -x NUM_NODES=4
exec hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]' -ginkgo.skip='\[Skipped\]' ${TEST_ARGS}
exec hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]' ${TEST_ARGS}

39
hack/jenkins/e2e.sh
View File

@@ -225,8 +225,7 @@ case ${JOB_NAME} in
: ${E2E_NETWORK:="e2e-gce-${NODE_NAME}-${EXECUTOR_NUMBER}"}
: ${GINKGO_PARALLEL:="y"}
# This list should match the list in kubernetes-e2e-gce.
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${KUBE_GCE_INSTANCE_PREFIX:="e2e-gce-${NODE_NAME}-${EXECUTOR_NUMBER}"}
: ${PROJECT:="kubernetes-jenkins-pull"}
# Override GCE defaults
@@ -241,8 +240,7 @@ case ${JOB_NAME} in
: ${E2E_PUBLISH_GREEN_VERSION:="true"}
: ${E2E_NETWORK:="e2e-gce"}
# This list should match the list in kubernetes-pull-build-test-e2e-gce.
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${GINKGO_PARALLEL:="y"}
: ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce"}
: ${PROJECT:="k8s-jkns-e2e-gce"}
@@ -253,9 +251,8 @@ case ${JOB_NAME} in
kubernetes-e2e-gce-slow)
: ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-slow"}
: ${E2E_NETWORK:="e2e-slow"}
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Slow\] \
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${GINKGO_PARALLEL:="y"}
: ${KUBE_GCE_INSTANCE_PREFIX:="e2e-slow"}
: ${PROJECT:="k8s-jkns-e2e-gce-slow"}
@@ -304,9 +301,8 @@ case ${JOB_NAME} in
kubernetes-e2e-gce-flaky)
: ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-flaky"}
: ${E2E_NETWORK:="e2e-flaky"}
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \
--ginkgo.skip=\[Feature:.+\]|\[Skipped\]"}
--ginkgo.skip=\[Feature:.+\]"}
: ${KUBE_GCE_INSTANCE_PREFIX:="e2e-flaky"}
: ${PROJECT:="k8s-jkns-e2e-gce-flaky"}
: ${FAIL_ON_GCP_RESOURCE_LEAK:="true"}
@@ -318,9 +314,8 @@ case ${JOB_NAME} in
: ${E2E_CLUSTER_NAME:="parallel-flaky"}
: ${E2E_NETWORK:="e2e-parallel-flaky"}
: ${GINKGO_PARALLEL:="y"}
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Feature:.+\]|\[Skipped\]"}
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Feature:.+\]"}
: ${KUBE_GCE_INSTANCE_PREFIX:="parallel-flaky"}
: ${PROJECT:="k8s-jkns-e2e-gce-prl-flaky"}
: ${FAIL_ON_GCP_RESOURCE_LEAK:="true"}
@@ -337,8 +332,7 @@ case ${JOB_NAME} in
: ${E2E_SET_CLUSTER_API_VERSION:=y}
: ${PROJECT:="k8s-jkns-e2e-gke-ci"}
: ${FAIL_ON_GCP_RESOURCE_LEAK:="true"}
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${GINKGO_PARALLEL:="y"}
;;
@@ -348,9 +342,8 @@ case ${JOB_NAME} in
: ${E2E_SET_CLUSTER_API_VERSION:=y}
: ${PROJECT:="k8s-jkns-e2e-gke-slow"}
: ${FAIL_ON_GCP_RESOURCE_LEAK:="true"}
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Slow\] \
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${GINKGO_PARALLEL:="y"}
;;
@@ -371,9 +364,8 @@ case ${JOB_NAME} in
: ${E2E_SET_CLUSTER_API_VERSION:=y}
: ${PROJECT:="k8s-jkns-e2e-gke-ci-flaky"}
: ${FAIL_ON_GCP_RESOURCE_LEAK:="true"}
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \
--ginkgo.skip=\[Feature:.+\]|\[Skipped\]"}
--ginkgo.skip=\[Feature:.+\]"}
;;
# AWS core jobs
@@ -385,8 +377,7 @@ case ${JOB_NAME} in
: ${E2E_ZONE:="us-west-2a"}
: ${ZONE:="us-west-2a"}
: ${E2E_NETWORK:="e2e-aws"}
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${GINKGO_PARALLEL:="y"}
: ${KUBE_GCE_INSTANCE_PREFIX="e2e-aws"}
: ${PROJECT:="k8s-jkns-e2e-aws"}
@@ -619,9 +610,7 @@ case ${JOB_NAME} in
# We should be testing the reliability of a long-running cluster. The
# [Disruptive] tests kill/restart components or nodes in the cluster,
# defeating the purpose of a soak cluster. (#15722)
#
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly"}
: ${HAIRPIN_MODE:="false"}
: ${PROJECT:="kubernetes-jenkins"}
@@ -649,9 +638,7 @@ case ${JOB_NAME} in
# We should be testing the reliability of a long-running cluster. The
# [Disruptive] tests kill/restart components or nodes in the cluster,
# defeating the purpose of a soak cluster. (#15722)
#
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
: ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly-2"}
: ${PROJECT:="kubernetes-jenkins"}
;;
@@ -683,9 +670,7 @@ case ${JOB_NAME} in
# We should be testing the reliability of a long-running cluster. The
# [Disruptive] tests kill/restart components or nodes in the cluster,
# defeating the purpose of a soak cluster. (#15722)
#
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"}
;;
# Upgrade jobs

3
test/e2e/e2e.go
View File

@@ -221,8 +221,7 @@ func RunE2ETests(t *testing.T) {
gomega.RegisterFailHandler(ginkgo.Fail)
// Disable skipped tests unless they are explicitly requested.
if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
// TODO(ihmccreery) Remove [Skipped] once all [Skipped] labels have been reclassified.
config.GinkgoConfig.SkipString = `\[Flaky\]|\[Skipped\]|\[Feature:.+\]`
config.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]`
}
// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins