github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add integration tests with failpoint

Browse Source 
Signed-off-by: Qiutong Song <songqt01@gmail.com>
This commit is contained in:
Qiutong Song 2022-09-27 14:39:07 +00:00
parent 4f4aad057d
commit 4196fad027
2 changed files with 152 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
integration/main_test.go
View File

@ -164,6 +164,15 @@ func WithPodHostname(hostname string) PodSandboxOpts {
}
}
// Add pod labels.
func WithPodLabels(kvs map[string]string) PodSandboxOpts {
return func(p *runtime.PodSandboxConfig) {
for k, v := range kvs {
p.Labels[k] = v
}
}
}
// PodSandboxConfig generates a pod sandbox config for test.
func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandboxConfig {
config := &runtime.PodSandboxConfig{
@ -176,6 +185,7 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb
},
Linux: &runtime.LinuxPodSandboxConfig{},
Annotations: make(map[string]string),
Labels: make(map[string]string),
}
for _, opt := range opts {
opt(config)

144
integration/sandbox_run_rollback_test.go
View File

@ -28,10 +28,10 @@ import (
"strings"
"testing"
"github.com/stretchr/testify/require"
criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/failpoint"
"github.com/stretchr/testify/require"
)
const (
@ -89,6 +89,146 @@ func TestRunPodSandboxWithShimStartFailure(t *testing.T) {
require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings"))
}
// TestRunPodSandboxWithShimDeleteFailure should keep the sandbox record if
// failed to rollback shim by shim.Delete API.
func TestRunPodSandboxWithShimDeleteFailure(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip()
}
if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
t.Skip()
}
testCase := func(restart bool) func(*testing.T) {
return func(t *testing.T) {
t.Log("Init PodSandboxConfig with specific label")
labels := map[string]string{
t.Name(): "true",
}
sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))
t.Log("Inject Shim failpoint")
injectShimFailpoint(t, sbConfig, map[string]string{
"Start": "1*error(failed to start shim)",
"Delete": "1*error(please retry)", // inject failpoint during rollback shim
})
t.Log("Create a sandbox")
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
require.Error(t, err)
require.ErrorContains(t, err, "failed to start shim")
t.Log("ListPodSandbox with the specific label")
l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
require.NoError(t, err)
require.Len(t, l, 1)
sb := l[0]
require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name)
require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace)
require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
t.Log("Check PodSandboxStatus")
sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
require.NoError(t, err)
require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
require.Greater(t, len(sbStatus.Network.Ip), 0)
if restart {
t.Log("Restart containerd")
RestartContainerd(t)
t.Log("ListPodSandbox with the specific label")
l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
require.NoError(t, err)
require.Len(t, l, 1)
require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
t.Log("Check PodSandboxStatus")
sbStatus, err := runtimeService.PodSandboxStatus(sb.Id)
require.NoError(t, err)
t.Log(sbStatus.Network)
require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
}
t.Log("Cleanup leaky sandbox")
err = runtimeService.RemovePodSandbox(sb.Id)
require.NoError(t, err)
}
}
t.Run("CleanupAfterRestart", testCase(true))
t.Run("JustCleanup", testCase(false))
}
// TestRunPodSandboxWithShimStartAndTeardownCNIFailure should keep the sandbox
// record if failed to rollback CNI API.
func TestRunPodSandboxWithShimStartAndTeardownCNIFailure(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip()
}
if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
t.Skip()
}
testCase := func(restart bool) func(*testing.T) {
return func(t *testing.T) {
t.Log("Init PodSandboxConfig with specific key")
labels := map[string]string{
t.Name(): "true",
}
sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels))
t.Log("Inject Shim failpoint")
injectShimFailpoint(t, sbConfig, map[string]string{
"Start": "1*error(failed to start shim)",
})
t.Log("Inject CNI failpoint")
conf := &failpointConf{
Del: "1*error(please retry)",
}
injectCNIFailpoint(t, sbConfig, conf)
t.Log("Create a sandbox")
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
require.Error(t, err)
require.ErrorContains(t, err, "failed to start shim")
t.Log("ListPodSandbox with the specific label")
l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
require.NoError(t, err)
require.Len(t, l, 1)
sb := l[0]
require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name)
require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace)
require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
if restart {
t.Log("Restart containerd")
RestartContainerd(t)
t.Log("ListPodSandbox with the specific label")
l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
require.NoError(t, err)
require.Len(t, l, 1)
require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
}
t.Log("Cleanup leaky sandbox")
err = runtimeService.RemovePodSandbox(sb.Id)
require.NoError(t, err)
}
}
t.Run("CleanupAfterRestart", testCase(true))
t.Run("JustCleanup", testCase(false))
}
// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
type failpointConf struct {
Add string `json:"cmdAdd"`
@ -101,7 +241,7 @@ func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf
metadata := sbConfig.Metadata
fpFilename := filepath.Join(stateDir,
fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name))
fmt.Sprintf("%s-%s.json", metadata.Namespace, strings.Replace(metadata.Name, "/", "-", -1)))
data, err := json.Marshal(conf)
require.NoError(t, err)