Mark relevant tests as elligible for parallelism

Signed-off-by: Kenfe-Mickael Laventure <mickael.laventure@gmail.com>

Makefile

@@ -69,7 +69,8 @@ ifeq ($(filter \
 endif
 
 # Flags passed to `go test`
-TESTFLAGS ?=-parallel 8 -v $(TESTFLAGS_RACE)
+TESTFLAGS ?= -v $(TESTFLAGS_RACE)
+TESTFLAGS_PARALLEL ?= 8
 
 .PHONY: clean all AUTHORS fmt vet lint dco build binaries test integration setup generate protos checkprotos coverage ci check help install uninstall vendor release
 .DEFAULT: default
@@ -154,7 +155,11 @@ root-test: ## run tests, except integration tests
 
 integration: ## run integration tests
 	@echo "$(WHALE) $@"
-	@go test ${TESTFLAGS} -test.root
+	@go test ${TESTFLAGS} -test.root -parallel 1
+
+integration-parallel: ## run integration tests
+	@echo "$(WHALE) $@"
+	@go test ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL}
 
 benchmark: ## run benchmarks tests
 	@echo "$(WHALE) $@"

client_test.go

@@ -149,6 +149,8 @@ func newClient(t testing.TB, address string, opts ...ClientOpt) (*Client, error)
 }
 
 func TestNewClient(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -161,6 +163,7 @@ func TestNewClient(t *testing.T) {
 	}
 }
 
+// All the container's tests depends on this, we need it to run first.
 func TestImagePull(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		// TODO: remove once Windows has a snapshotter

container_linux_test.go

@@ -15,6 +15,8 @@ import (
 )
 
 func TestContainerUpdate(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -104,6 +106,8 @@ func TestContainerUpdate(t *testing.T) {
 }
 
 func TestShimInCgroup(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)

container_test.go

@@ -44,6 +44,8 @@ func TestContainerList(t *testing.T) {
 }
 
 func TestNewContainer(t *testing.T) {
+	t.Parallel()
+
 	id := t.Name()
 	client, err := newClient(t, address)
 	if err != nil {
@@ -80,6 +82,8 @@ func TestNewContainer(t *testing.T) {
 }
 
 func TestContainerStart(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -151,6 +155,8 @@ func TestContainerStart(t *testing.T) {
 }
 
 func TestContainerOutput(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -225,6 +231,8 @@ func TestContainerOutput(t *testing.T) {
 }
 
 func TestContainerExec(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -322,6 +330,8 @@ func TestContainerExec(t *testing.T) {
 }
 
 func TestContainerPids(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -401,6 +411,8 @@ func TestContainerPids(t *testing.T) {
 }
 
 func TestContainerCloseIO(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -492,6 +504,8 @@ func TestContainerCloseIO(t *testing.T) {
 }
 
 func TestContainerAttach(t *testing.T) {
+	t.Parallel()
+
 	if runtime.GOOS == "windows" {
 		// On windows, closing the write side of the pipe closes the read
 		// side, sending an EOF to it and preventing reopening it.
@@ -625,6 +639,8 @@ func TestContainerAttach(t *testing.T) {
 }
 
 func TestDeleteRunningContainer(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -694,6 +710,8 @@ func TestDeleteRunningContainer(t *testing.T) {
 }
 
 func TestContainerKill(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -764,6 +782,8 @@ func TestContainerKill(t *testing.T) {
 }
 
 func TestContainerNoBinaryExists(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -816,6 +836,8 @@ func TestContainerNoBinaryExists(t *testing.T) {
 }
 
 func TestContainerExecNoBinaryExists(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -891,6 +913,8 @@ func TestContainerExecNoBinaryExists(t *testing.T) {
 }
 
 func TestUserNamespaces(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -969,6 +993,8 @@ func TestUserNamespaces(t *testing.T) {
 }
 
 func TestWaitStoppedTask(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -1039,6 +1065,8 @@ func TestWaitStoppedTask(t *testing.T) {
 }
 
 func TestWaitStoppedProcess(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -1134,6 +1162,8 @@ func TestWaitStoppedProcess(t *testing.T) {
 }
 
 func TestTaskForceDelete(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -1185,6 +1215,8 @@ func TestTaskForceDelete(t *testing.T) {
 }
 
 func TestProcessForceDelete(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
@@ -1264,6 +1296,8 @@ func TestProcessForceDelete(t *testing.T) {
 }
 
 func TestContainerHostname(t *testing.T) {
+	t.Parallel()
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)

snapshot/testsuite/issues.go

@@ -22,6 +22,8 @@ import (
 // avoid such issues by not relying on tar to create layers.
 // See https://github.com/docker/docker/issues/21555
 func checkLayerFileUpdate(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	l1Init := fstest.Apply(
 		fstest.CreateDir("/etc", 0700),
 		fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
@@ -53,6 +55,8 @@ func checkLayerFileUpdate(ctx context.Context, t *testing.T, sn snapshot.Snapsho
 // checkRemoveDirectoryInLowerLayer
 // See https://github.com/docker/docker/issues/25244
 func checkRemoveDirectoryInLowerLayer(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	l1Init := fstest.Apply(
 		fstest.CreateDir("/lib", 0700),
 		fstest.CreateFile("/lib/hidden", []byte{}, 0644),
@@ -76,6 +80,8 @@ func checkRemoveDirectoryInLowerLayer(ctx context.Context, t *testing.T, sn snap
 // See https://github.com/docker/docker/issues/24913 overlay
 // see https://github.com/docker/docker/issues/28391 overlay2
 func checkChown(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	l1Init := fstest.Apply(
 		fstest.CreateDir("/opt", 0700),
 		fstest.CreateDir("/opt/a", 0700),
@@ -118,6 +124,8 @@ func checkRename(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, wor
 // checkDirectoryPermissionOnCommit
 // https://github.com/docker/docker/issues/27298
 func checkDirectoryPermissionOnCommit(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	l1Init := fstest.Apply(
 		fstest.CreateDir("/dir1", 0700),
 		fstest.CreateDir("/dir2", 0700),

snapshot/testsuite/testsuite.go

@@ -20,24 +20,26 @@ import (
 
 // SnapshotterSuite runs a test suite on the snapshotter given a factory function.
 func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error)) {
-	t.Run("Basic", makeTest(t, name, snapshotterFn, checkSnapshotterBasic))
+	t.Parallel()
-	t.Run("StatActive", makeTest(t, name, snapshotterFn, checkSnapshotterStatActive))
-	t.Run("StatComitted", makeTest(t, name, snapshotterFn, checkSnapshotterStatCommitted))
-	t.Run("TransitivityTest", makeTest(t, name, snapshotterFn, checkSnapshotterTransitivity))
-	t.Run("PreareViewFailingtest", makeTest(t, name, snapshotterFn, checkSnapshotterPrepareView))
-	t.Run("Update", makeTest(t, name, snapshotterFn, checkUpdate))
-	t.Run("Remove", makeTest(t, name, snapshotterFn, checkRemove))
 
-	t.Run("LayerFileupdate", makeTest(t, name, snapshotterFn, checkLayerFileUpdate))
+	t.Run("Basic", makeTest(name, snapshotterFn, checkSnapshotterBasic))
-	t.Run("RemoveDirectoryInLowerLayer", makeTest(t, name, snapshotterFn, checkRemoveDirectoryInLowerLayer))
+	t.Run("StatActive", makeTest(name, snapshotterFn, checkSnapshotterStatActive))
-	t.Run("Chown", makeTest(t, name, snapshotterFn, checkChown))
+	t.Run("StatComitted", makeTest(name, snapshotterFn, checkSnapshotterStatCommitted))
-	t.Run("DirectoryPermissionOnCommit", makeTest(t, name, snapshotterFn, checkDirectoryPermissionOnCommit))
+	t.Run("TransitivityTest", makeTest(name, snapshotterFn, checkSnapshotterTransitivity))
+	t.Run("PreareViewFailingtest", makeTest(name, snapshotterFn, checkSnapshotterPrepareView))
+	t.Run("Update", makeTest(name, snapshotterFn, checkUpdate))
+	t.Run("Remove", makeTest(name, snapshotterFn, checkRemove))
+
+	t.Run("LayerFileupdate", makeTest(name, snapshotterFn, checkLayerFileUpdate))
+	t.Run("RemoveDirectoryInLowerLayer", makeTest(name, snapshotterFn, checkRemoveDirectoryInLowerLayer))
+	t.Run("Chown", makeTest(name, snapshotterFn, checkChown))
+	t.Run("DirectoryPermissionOnCommit", makeTest(name, snapshotterFn, checkDirectoryPermissionOnCommit))
 
 	// Rename test still fails on some kernels with overlay
-	//t.Run("Rename", makeTest(t, name, snapshotterFn, checkRename))
+	//t.Run("Rename", makeTest(name, snapshotterFn, checkRename))
 }
 
-func makeTest(t *testing.T, name string, snapshotterFn func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error), fn func(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string)) func(t *testing.T) {
+func makeTest(name string, snapshotterFn func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error), fn func(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string)) func(t *testing.T) {
 	return func(t *testing.T) {
 		ctx := context.Background()
 		ctx = namespaces.WithNamespace(ctx, "testsuite")
@@ -78,6 +80,9 @@ func makeTest(t *testing.T, name string, snapshotterFn func(ctx context.Context,
 
 // checkSnapshotterBasic tests the basic workflow of a snapshot snapshotter.
 func checkSnapshotterBasic(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
+	// TODO: this always fails when run in parallel, why?
+	//	t.Parallel()
+
 	initialApplier := fstest.Apply(
 		fstest.CreateFile("/foo", []byte("foo\n"), 0777),
 		fstest.CreateDir("/a", 0755),
@@ -211,6 +216,8 @@ func checkSnapshotterBasic(ctx context.Context, t *testing.T, snapshotter snapsh
 
 // Create a New Layer on top of base layer with Prepare, Stat on new layer, should return Active layer.
 func checkSnapshotterStatActive(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	preparing := filepath.Join(work, "preparing")
 	if err := os.MkdirAll(preparing, 0777); err != nil {
 		t.Fatal(err)
@@ -245,6 +252,8 @@ func checkSnapshotterStatActive(ctx context.Context, t *testing.T, snapshotter s
 
 // Commit a New Layer on top of base layer with Prepare & Commit , Stat on new layer, should return Committed layer.
 func checkSnapshotterStatCommitted(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	preparing := filepath.Join(work, "preparing")
 	if err := os.MkdirAll(preparing, 0777); err != nil {
 		t.Fatal(err)
@@ -306,6 +315,8 @@ func snapshotterPrepareMount(ctx context.Context, snapshotter snapshot.Snapshott
 
 // Given A <- B <- C, B is the parent of C and A is a transitive parent of C (in this case, a "grandparent")
 func checkSnapshotterTransitivity(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	preparing, err := snapshotterPrepareMount(ctx, snapshotter, "preparing", "", work)
 	if err != nil {
 		t.Fatal(err)
@@ -360,6 +371,8 @@ func checkSnapshotterTransitivity(ctx context.Context, t *testing.T, snapshotter
 
 // Creating two layers with Prepare or View with same key must fail.
 func checkSnapshotterPrepareView(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	preparing, err := snapshotterPrepareMount(ctx, snapshotter, "preparing", "", work)
 	if err != nil {
 		t.Fatal(err)
@@ -456,6 +469,8 @@ func baseTestSnapshots(ctx context.Context, snapshotter snapshot.Snapshotter) er
 }
 
 func checkUpdate(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	t1 := time.Now().UTC()
 	if err := baseTestSnapshots(ctx, snapshotter); err != nil {
 		t.Fatalf("Failed to create base snapshots: %v", err)
@@ -607,6 +622,8 @@ func assertLabels(t *testing.T, actual, expected map[string]string) {
 }
 
 func checkRemove(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
+	t.Parallel()
+
 	if _, err := snapshotter.Prepare(ctx, "committed-a", ""); err != nil {
 		t.Fatal(err)
 	}

spec_unix_test.go

@@ -9,6 +9,8 @@ import (
 )
 
 func TestGenerateSpec(t *testing.T) {
+	t.Parallel()
+
 	s, err := GenerateSpec()
 	if err != nil {
 		t.Fatal(err)
@@ -47,6 +49,8 @@ func TestGenerateSpec(t *testing.T) {
 }
 
 func TestSpecWithTTY(t *testing.T) {
+	t.Parallel()
+
 	s, err := GenerateSpec(WithTTY)
 	if err != nil {
 		t.Fatal(err)
@@ -61,6 +65,8 @@ func TestSpecWithTTY(t *testing.T) {
 }
 
 func TestWithLinuxNamespace(t *testing.T) {
+	t.Parallel()
+
 	replacedNS := specs.LinuxNamespace{Type: specs.NetworkNamespace, Path: "/var/run/netns/test"}
 	s, err := GenerateSpec(WithLinuxNamespace(replacedNS))
 	if err != nil {