Merge pull request #7892 from AkihiroSuda/cri-integration-cgroup2

Cirrus CI (Fedora 37, Rocky 8): enable cri-integration

.cirrus.yml

@@ -40,6 +40,9 @@ task:
   integration_script: |
     vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
 
+  cri_integration_script: |
+    vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
+
   cri_test_script: |
     vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
 

Vagrantfile

@@ -207,6 +207,19 @@ EOF
       SHELL
   end
 
+  config.vm.provision "install-failpoint-binaries", type: "shell",  run: "once" do |sh|
+      sh.upload_path = "/tmp/vagrant-install-failpoint-binaries"
+      sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-failpoint-binaries
+        chcon -v -t container_runtime_exec_t $(type -ap containerd-shim-runc-fp-v1)
+        containerd-shim-runc-fp-v1 -v
+      SHELL
+  end
+
   # SELinux is Enforcing by default.
   # To set SELinux as Disabled on a VM that has already been provisioned:
   #   SELINUX=Disabled vagrant up --provision-with=selinux
@@ -246,6 +259,36 @@ EOF
     SHELL
   end
 
+  # SELinux is Enforcing by default (via provisioning) in this VM. To re-run with SELinux disabled:
+  #   SELINUX=Disabled vagrant up --provision-with=selinux,test-cri-integration
+  #
+  config.vm.provision "test-cri-integration", type: "shell", run: "never" do |sh|
+    sh.upload_path = "/tmp/test-cri-integration"
+    sh.env = {
+        'GOTEST': ENV['GOTEST'] || "go test",
+        'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
+        'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'],
+        'GITHUB_WORKSPACE': '',
+        'ENABLE_CRI_SANDBOXES': ENV['ENABLE_CRI_SANDBOXES'],
+    }
+    sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        cleanup() {
+          rm -rf /var/lib/containerd* /run/containerd* /tmp/containerd* /tmp/test* /tmp/failpoint* /tmp/nri*
+        }
+        cleanup
+        cd ${GOPATH}/src/github.com/containerd/containerd
+        # cri-integration.sh executes containerd from ./bin, not from $PATH .
+        make BUILDTAGS="seccomp selinux no_aufs no_btrfs no_devmapper no_zfs" binaries bin/cri-integration.test
+        chcon -v -t container_runtime_exec_t ./bin/{containerd,containerd-shim*}
+        CONTAINERD_RUNTIME=io.containerd.runc.v2 ./script/test/cri-integration.sh
+        cleanup
+    SHELL
+  end
+
   # SELinux is Enforcing by default (via provisioning) in this VM. To re-run with SELinux disabled:
   #   SELINUX=Disabled vagrant up --provision-with=selinux,test-cri
   #

integration/container_stats_test.go

@@ -61,7 +61,7 @@ func TestContainerStats(t *testing.T) {
 		if err != nil {
 			return false, err
 		}
-		if s.GetWritableLayer().GetUsedBytes().GetValue() != 0 {
+		if s.GetWritableLayer().GetTimestamp() != 0 {
 			return true, nil
 		}
 		return false, nil
@@ -103,7 +103,7 @@ func TestContainerConsumedStats(t *testing.T) {
 		if err != nil {
 			return false, err
 		}
-		if s.GetMemory().GetWorkingSetBytes().GetValue() > 0 {
+		if s.GetWritableLayer().GetTimestamp() > 0 {
 			return true, nil
 		}
 		return false, nil
@@ -179,7 +179,7 @@ func TestContainerListStats(t *testing.T) {
 			return false, err
 		}
 		for _, s := range stats {
-			if s.GetWritableLayer().GetUsedBytes().GetValue() == 0 {
+			if s.GetWritableLayer().GetTimestamp() == 0 {
 				return false, nil
 			}
 		}
@@ -238,7 +238,7 @@ func TestContainerListStatsWithIdFilter(t *testing.T) {
 			if len(stats) != 1 {
 				return false, errors.New("unexpected stats length")
 			}
-			if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 {
+			if stats[0].GetWritableLayer().GetTimestamp() != 0 {
 				return true, nil
 			}
 			return false, nil
@@ -300,7 +300,7 @@ func TestContainerListStatsWithSandboxIdFilter(t *testing.T) {
 
 		for _, containerStats := range stats {
 			// Wait for stats on all containers, not just the first one in the list.
-			if containerStats.GetWritableLayer().GetUsedBytes().GetValue() == 0 {
+			if containerStats.GetWritableLayer().GetTimestamp() == 0 {
 				return false, nil
 			}
 		}
@@ -358,7 +358,7 @@ func TestContainerListStatsWithIdSandboxIdFilter(t *testing.T) {
 			if len(stats) != 1 {
 				return false, errors.New("unexpected stats length")
 			}
-			if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 {
+			if stats[0].GetWritableLayer().GetTimestamp() != 0 {
 				return true, nil
 			}
 			return false, nil
@@ -380,7 +380,7 @@ func TestContainerListStatsWithIdSandboxIdFilter(t *testing.T) {
 			if len(stats) != 1 {
 				return false, fmt.Errorf("expected only one stat, but got %v", stats)
 			}
-			if stats[0].GetWritableLayer().GetUsedBytes().GetValue() != 0 {
+			if stats[0].GetWritableLayer().GetTimestamp() != 0 {
 				return true, nil
 			}
 			return false, nil
@@ -410,7 +410,12 @@ func testStats(t *testing.T,
 	require.NotEmpty(t, s.GetMemory().GetWorkingSetBytes().GetValue())
 	require.NotEmpty(t, s.GetWritableLayer().GetTimestamp())
 	require.NotEmpty(t, s.GetWritableLayer().GetFsId().GetMountpoint())
-	require.NotEmpty(t, s.GetWritableLayer().GetUsedBytes().GetValue())
+
+	// UsedBytes of a fresh container can be zero on Linux, depending on the backing filesystem.
+	// https://github.com/containerd/containerd/issues/7909
+	if goruntime.GOOS == "windows" {
+		require.NotEmpty(t, s.GetWritableLayer().GetUsedBytes().GetValue())
+	}
 
 	// Windows does not collect inodes stats.
 	if goruntime.GOOS != "windows" {

integration/main_test.go

@@ -43,6 +43,7 @@ import (
 	"github.com/containerd/containerd/pkg/cri/constants"
 	"github.com/containerd/containerd/pkg/cri/server"
 	"github.com/containerd/containerd/pkg/cri/util"
+	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -289,7 +290,11 @@ func WithVolumeMount(hostPath, containerPath string) ContainerOpts {
 	return func(c *runtime.ContainerConfig) {
 		hostPath, _ = filepath.Abs(hostPath)
 		containerPath, _ = filepath.Abs(containerPath)
-		mount := &runtime.Mount{HostPath: hostPath, ContainerPath: containerPath}
+		mount := &runtime.Mount{
+			HostPath:       hostPath,
+			ContainerPath:  containerPath,
+			SelinuxRelabel: selinux.GetEnabled(),
+		}
 		c.Mounts = append(c.Mounts, mount)
 	}
 }

integration/nri_test.go

@@ -32,6 +32,7 @@ import (
 	cri "github.com/containerd/containerd/integration/cri-api/pkg/apis"
 	"github.com/containerd/nri/pkg/api"
 	"github.com/containerd/nri/pkg/stub"
+	"github.com/opencontainers/selinux/go-selinux"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 
 	"github.com/containerd/containerd/integration/images"
@@ -57,6 +58,11 @@ func skipNriTestIfNecessary(t *testing.T, extraSkipChecks ...map[string]bool) {
 	if goruntime.GOOS != "linux" {
 		t.Skip("Not running on linux")
 	}
+
+	if selinux.GetEnabled() {
+		// https://github.com/containerd/containerd/pull/7892#issuecomment-1369825603
+		t.Skip("SELinux relabeling is not supported for NRI yet")
+	}
 	_, err := os.Stat(nriTestSocket)
 	if err != nil {
 		t.Skip("Containerd test instance does not have NRI enabled")

integration/volume_copy_up_test.go

@@ -118,17 +118,17 @@ func TestVolumeOwnership(t *testing.T) {
 	require.NoError(t, runtimeService.StartContainer(cn))
 
 	// ghcr.io/containerd/volume-ownership:2.1 contains a test_dir
-	// volume, which is owned by nobody:nogroup.
+	// volume, which is owned by 65534:65534 (nobody:nogroup, or nobody:nobody).
 	// On Windows, the folder is situated in C:\volumes\test_dir and is owned
 	// by ContainerUser (SID: S-1-5-93-2-2). A helper tool get_owner.exe should
 	// exist inside the container that returns the owner in the form of USERNAME:SID.
 	t.Logf("Check ownership of test directory inside container")
 
 	cmd := []string{
-		"stat", "-c", "%U:%G", "/test_dir",
+		"stat", "-c", "%u:%g", "/test_dir",
 	}
-	expectedContainerOutput := "nobody:nogroup\n"
-	expectedHostOutput := "nobody:nogroup\n"
+	expectedContainerOutput := "65534:65534\n"
+	expectedHostOutput := "65534:65534\n"
 	if goruntime.GOOS == "windows" {
 		cmd = []string{
 			"C:\\bin\\get_owner.exe",

integration/volume_copy_up_unix_test.go

@@ -25,7 +25,7 @@ import (
 )
 
 func getOwnership(path string) (string, error) {
-	hostCmd := fmt.Sprintf("stat -c %%U:%%G '%s'", path)
+	hostCmd := fmt.Sprintf("stat -c %%u:%%g '%s'", path)
 	output, err := exec.Command("sh", "-c", hostCmd).CombinedOutput()
 	if err != nil {
 		return "", err