Add integration tests with failpoint

Signed-off-by: Qiutong Song <songqt01@gmail.com>

integration/main_test.go

@@ -164,6 +164,15 @@ func WithPodHostname(hostname string) PodSandboxOpts {
 	}
 }
 
+// Add pod labels.
+func WithPodLabels(kvs map[string]string) PodSandboxOpts {
+	return func(p *runtime.PodSandboxConfig) {
+		for k, v := range kvs {
+			p.Labels[k] = v
+		}
+	}
+}
+
 // PodSandboxConfig generates a pod sandbox config for test.
 func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandboxConfig {
 	config := &runtime.PodSandboxConfig{
@@ -176,6 +185,7 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb
 		},
 		Linux:       &runtime.LinuxPodSandboxConfig{},
 		Annotations: make(map[string]string),
+		Labels:      make(map[string]string),
 	}
 	for _, opt := range opts {
 		opt(config)

integration/sandbox_run_rollback_test.go

@@ -28,10 +28,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/require"
 	criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
 
 	"github.com/containerd/containerd/pkg/failpoint"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -89,6 +89,146 @@ func TestRunPodSandboxWithShimStartFailure(t *testing.T) {
 	require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings"))
 }
 
+// TestRunPodSandboxWithShimDeleteFailure should keep the sandbox record if
+// failed to rollback shim by shim.Delete API.
+func TestRunPodSandboxWithShimDeleteFailure(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip()
+	}
+	if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
+		t.Skip()
+	}
+
+	testCase := func(restart bool) func(*testing.T) {
+		return func(t *testing.T) {
+			t.Log("Init PodSandboxConfig with specific label")
+			labels := map[string]string{
+				t.Name(): "true",
+			}
+			sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))
+
+			t.Log("Inject Shim failpoint")
+			injectShimFailpoint(t, sbConfig, map[string]string{
+				"Start":  "1*error(failed to start shim)",
+				"Delete": "1*error(please retry)", // inject failpoint during rollback shim
+			})
+
+			t.Log("Create a sandbox")
+			_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
+			require.Error(t, err)
+			require.ErrorContains(t, err, "failed to start shim")
+
+			t.Log("ListPodSandbox with the specific label")
+			l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
+			require.NoError(t, err)
+			require.Len(t, l, 1)
+
+			sb := l[0]
+			require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
+			require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name)
+			require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace)
+			require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
+			require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
+
+			t.Log("Check PodSandboxStatus")
+			sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
+			require.NoError(t, err)
+			require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
+			require.Greater(t, len(sbStatus.Network.Ip), 0)
+
+			if restart {
+				t.Log("Restart containerd")
+				RestartContainerd(t)
+
+				t.Log("ListPodSandbox with the specific label")
+				l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
+				require.NoError(t, err)
+				require.Len(t, l, 1)
+				require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
+
+				t.Log("Check PodSandboxStatus")
+				sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
+				require.NoError(t, err)
+				t.Log(sbStatus.Network)
+				require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
+			}
+
+			t.Log("Cleanup leaky sandbox")
+			err = runtimeService.RemovePodSandbox(sb.Id)
+			require.NoError(t, err)
+		}
+	}
+
+	t.Run("CleanupAfterRestart", testCase(true))
+	t.Run("JustCleanup", testCase(false))
+}
+
+// TestRunPodSandboxWithShimStartAndTeardownCNIFailure should keep the sandbox
+// record if failed to rollback CNI API.
+func TestRunPodSandboxWithShimStartAndTeardownCNIFailure(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip()
+	}
+	if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
+		t.Skip()
+	}
+
+	testCase := func(restart bool) func(*testing.T) {
+		return func(t *testing.T) {
+			t.Log("Init PodSandboxConfig with specific key")
+			labels := map[string]string{
+				t.Name(): "true",
+			}
+			sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))
+
+			t.Log("Inject Shim failpoint")
+			injectShimFailpoint(t, sbConfig, map[string]string{
+				"Start": "1*error(failed to start shim)",
+			})
+
+			t.Log("Inject CNI failpoint")
+			conf := &failpointConf{
+				Del: "1*error(please retry)",
+			}
+			injectCNIFailpoint(t, sbConfig, conf)
+
+			t.Log("Create a sandbox")
+			_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
+			require.Error(t, err)
+			require.ErrorContains(t, err, "failed to start shim")
+
+			t.Log("ListPodSandbox with the specific label")
+			l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
+			require.NoError(t, err)
+			require.Len(t, l, 1)
+
+			sb := l[0]
+			require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
+			require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name)
+			require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace)
+			require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
+			require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
+
+			if restart {
+				t.Log("Restart containerd")
+				RestartContainerd(t)
+
+				t.Log("ListPodSandbox with the specific label")
+				l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
+				require.NoError(t, err)
+				require.Len(t, l, 1)
+				require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
+			}
+
+			t.Log("Cleanup leaky sandbox")
+			err = runtimeService.RemovePodSandbox(sb.Id)
+			require.NoError(t, err)
+		}
+	}
+	t.Run("CleanupAfterRestart", testCase(true))
+	t.Run("JustCleanup", testCase(false))
+}
+
 // failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
 type failpointConf struct {
 	Add   string `json:"cmdAdd"`
@@ -101,7 +241,7 @@ func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf
 
 	metadata := sbConfig.Metadata
 	fpFilename := filepath.Join(stateDir,
-		fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name))
+		fmt.Sprintf("%s-%s.json", metadata.Namespace, strings.Replace(metadata.Name, "/", "-", -1)))
 
 	data, err := json.Marshal(conf)
 	require.NoError(t, err)