integration: Add injected failpoint testing for RunPodSandbox

Signed-off-by: Wei Fu <fuweid89@gmail.com>

.github/workflows/ci.yml

@@ -425,6 +425,10 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y criu
 
+      - name: Install failpoint binaries
+        run: |
+          script/setup/install-failpoint-binaries
+
       - name: Install containerd
         env:
           CGO_ENABLED: 1

integration/main_test.go

@@ -175,6 +175,7 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb
 			Namespace: Randomize(ns),
 		},
 		Linux:       &runtime.LinuxPodSandboxConfig{},
+		Annotations: make(map[string]string),
 	}
 	for _, opt := range opts {
 		opt(config)

integration/sandbox_run_rollback_test.go

@@ -0,0 +1,122 @@
+//go:build linux
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"github.com/containerd/containerd/pkg/failpoint"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	failpointRuntimeHandler = "runc-fp"
+
+	failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint."
+
+	failpointCNIStateDirKey = "cniFailpointControlStateDir"
+)
+
+func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip()
+	}
+
+	t.Logf("Inject CNI failpoint")
+	conf := &failpointConf{
+		Add: "1*error(you-shall-not-pass!)",
+	}
+
+	sbConfig := PodSandboxConfig(t.Name(), "failpoint")
+	injectCNIFailpoint(t, sbConfig, conf)
+
+	t.Logf("Create a sandbox")
+	_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
+	require.Error(t, err)
+	require.Equal(t, true, strings.Contains(err.Error(), "you-shall-not-pass!"))
+
+	t.Logf("Retry to create sandbox with same config")
+	sb, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
+	require.NoError(t, err)
+
+	err = runtimeService.StopPodSandbox(sb)
+	require.NoError(t, err)
+
+	err = runtimeService.RemovePodSandbox(sb)
+	require.NoError(t, err)
+}
+
+func TestRunPodSandboxWithShimStartFailure(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip()
+	}
+
+	t.Logf("Inject Shim failpoint")
+
+	sbConfig := PodSandboxConfig(t.Name(), "failpoint")
+	injectShimFailpoint(t, sbConfig, map[string]string{
+		"Start": "1*error(no hard feelings)",
+	})
+
+	t.Logf("Create a sandbox")
+	_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
+	require.Error(t, err)
+	require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings"))
+}
+
+// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
+type failpointConf struct {
+	Add   string `json:"cmdAdd"`
+	Del   string `json:"cmdDel"`
+	Check string `json:"cmdCheck"`
+}
+
+func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf *failpointConf) {
+	stateDir := t.TempDir()
+
+	metadata := sbConfig.Metadata
+	fpFilename := filepath.Join(stateDir,
+		fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name))
+
+	data, err := json.Marshal(conf)
+	require.NoError(t, err)
+
+	err = os.WriteFile(fpFilename, data, 0666)
+	require.NoError(t, err)
+
+	sbConfig.Annotations[failpointCNIStateDirKey] = stateDir
+}
+
+func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) {
+	for method, fp := range methodFps {
+		_, err := failpoint.NewFailpoint(method, fp)
+		require.NoError(t, err, "check failpoint %s for shim method %s", fp, method)
+
+		sbConfig.Annotations[failpointShimPrefixKey+method] = fp
+	}
+}

script/setup/install-failpoint-binaries

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# Build and install
+#
+# * cni-bridge-fp into /opt/cni/bin
+# * containerd-shim-runc-fp-v1 into /usr/local/bin
+#
+set -euo pipefail
+
+base_dir="$(dirname "${BASH_SOURCE[0]}")"
+root_dir="$( cd "${base_dir}" && pwd )"/../..
+
+cd "${root_dir}"
+
+CNI_BIN_DIR=${CNI_BIN_DIR:-"/opt/cni/bin"}
+make bin/cni-bridge-fp
+sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}"
+
+SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"}
+make bin/containerd-shim-runc-fp-v1
+sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}"

script/test/utils.sh

@@ -35,9 +35,10 @@ CONTAINERD_RUNTIME=${CONTAINERD_RUNTIME:-""}
 if [ -z "${CONTAINERD_CONFIG_FILE}" ]; then
   config_file="${CONTAINERD_CONFIG_DIR}/containerd-config-cri.toml"
   truncate --size 0 "${config_file}"
+  echo "version=2" >> ${config_file}
+
   if command -v sestatus >/dev/null 2>&1; then
     cat >>${config_file} <<EOF
-version=2
 [plugins."io.containerd.grpc.v1.cri"]
   enable_selinux = true
 EOF
@@ -51,6 +52,59 @@ EOF
   CONTAINERD_CONFIG_FILE="${config_file}"
 fi
 
+if [ $IS_WINDOWS -eq 0 ]; then
+  FAILPOINT_CONTAINERD_RUNTIME="runc-fp.v1"
+  FAILPOINT_CNI_CONF_DIR=${FAILPOINT_CNI_CONF_DIR:-"/tmp/failpoint-cni-net.d"}
+  mkdir -p "${FAILPOINT_CNI_CONF_DIR}"
+
+  # Add runtime with failpoint
+  cat << EOF | tee -a "${CONTAINERD_CONFIG_FILE}"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc-fp]
+  cni_conf_dir = "${FAILPOINT_CNI_CONF_DIR}"
+  cni_max_conf_num = 1
+  pod_annotations = ["io.containerd.runtime.v2.shim.failpoint.*"]
+  runtime_type = "${FAILPOINT_CONTAINERD_RUNTIME}"
+EOF
+
+  cat << EOF | tee "${FAILPOINT_CNI_CONF_DIR}/10-containerd-net.conflist"
+{
+  "cniVersion": "1.0.0",
+  "name": "containerd-net-failpoint",
+  "plugins": [
+    {
+      "type": "cni-bridge-fp",
+      "bridge": "cni-fp",
+      "isGateway": true,
+      "ipMasq": true,
+      "promiscMode": true,
+      "ipam": {
+        "type": "host-local",
+        "ranges": [
+          [{
+            "subnet": "10.88.0.0/16"
+          }],
+          [{
+            "subnet": "2001:4860:4860::/64"
+          }]
+        ],
+        "routes": [
+          { "dst": "0.0.0.0/0" },
+          { "dst": "::/0" }
+        ]
+      },
+      "capabilities": {
+        "io.kubernetes.cri.pod-annotations": true
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {"portMappings": true}
+    }
+  ]
+}
+EOF
+fi
+
 # CONTAINERD_TEST_SUFFIX is the suffix appended to the root/state directory used
 # by test containerd.
 CONTAINERD_TEST_SUFFIX=${CONTAINERD_TEST_SUFFIX:-"-test"}
@@ -183,6 +237,8 @@ test_setup() {
   fi
   readiness_check run_ctr
   readiness_check run_crictl
+  # Show the config about cri plugin in log when it's ready
+  run_crictl
 }
 
 # test_teardown kills containerd.