Update container with sandbox metadata after NetNS is created

Signed-off-by: Qiutong Song <songqt01@gmail.com>

integration/sandbox_run_rollback_test.go

@@ -26,16 +26,24 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
+	"syscall"
 	"testing"
+	"time"
 
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
 
+	"github.com/containerd/containerd/pkg/cri/store/sandbox"
 	"github.com/containerd/containerd/pkg/failpoint"
+	"github.com/containerd/typeurl"
 )
 
 const (
 	failpointRuntimeHandler = "runc-fp"
+	failpointCNIBinary      = "cni-bridge-fp"
 
 	failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint."
 
@@ -138,7 +146,7 @@ func TestRunPodSandboxWithShimDeleteFailure(t *testing.T) {
 
 			if restart {
 				t.Log("Restart containerd")
-				RestartContainerd(t)
+				RestartContainerd(t, syscall.SIGTERM)
 
 				t.Log("ListPodSandbox with the specific label")
 				l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
@@ -211,7 +219,7 @@ func TestRunPodSandboxWithShimStartAndTeardownCNIFailure(t *testing.T) {
 
 			if restart {
 				t.Log("Restart containerd")
-				RestartContainerd(t)
+				RestartContainerd(t, syscall.SIGTERM)
 
 				t.Log("ListPodSandbox with the specific label")
 				l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id})
@@ -229,6 +237,132 @@ func TestRunPodSandboxWithShimStartAndTeardownCNIFailure(t *testing.T) {
 	t.Run("JustCleanup", testCase(false))
 }
 
+// TestRunPodSandboxWithShimStartAndTeardownCNISlow should keep the sandbox
+// record if failed to rollback CNI API.
+func TestRunPodSandboxAndTeardownCNISlow(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip()
+	}
+	if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
+		t.Skip()
+	}
+
+	t.Log("Init PodSandboxConfig with specific key")
+	sbName := t.Name()
+	labels := map[string]string{
+		sbName: "true",
+	}
+	sbConfig := PodSandboxConfig(sbName, "failpoint", WithPodLabels(labels))
+
+	t.Log("Inject CNI failpoint")
+	conf := &failpointConf{
+		// Delay 1 day
+		Add: "1*delay(86400000)",
+	}
+	injectCNIFailpoint(t, sbConfig, conf)
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+		t.Log("Create a sandbox")
+		_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
+		require.Error(t, err)
+		require.ErrorContains(t, err, "error reading from server: EOF")
+	}()
+
+	assert.NoError(t, ensureCNIAddRunning(t, sbName), "check that failpoint CNI.Add is running")
+
+	// Use SIGKILL to prevent containerd server gracefulshutdown which may cause indeterministic invocation of defer functions
+	t.Log("Restart containerd")
+	RestartContainerd(t, syscall.SIGKILL)
+
+	wg.Wait()
+
+	t.Log("ListPodSandbox with the specific label")
+	l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels})
+	require.NoError(t, err)
+	require.Len(t, l, 1)
+
+	sb := l[0]
+
+	defer func() {
+		t.Log("Cleanup leaky sandbox")
+		err := runtimeService.StopPodSandbox(sb.Id)
+		assert.NoError(t, err)
+		err = runtimeService.RemovePodSandbox(sb.Id)
+		require.NoError(t, err)
+	}()
+
+	assert.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY)
+	assert.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name)
+	assert.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace)
+	assert.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid)
+	assert.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt)
+
+	t.Log("Get sandbox info")
+	_, info, err := SandboxInfo(sb.Id)
+	require.NoError(t, err)
+	require.False(t, info.NetNSClosed)
+
+	var netNS string
+	for _, n := range info.RuntimeSpec.Linux.Namespaces {
+		if n.Type == runtimespec.NetworkNamespace {
+			netNS = n.Path
+		}
+	}
+	assert.NotEmpty(t, netNS, "network namespace should be set")
+
+	t.Log("Get sandbox container")
+	c, err := GetContainer(sb.Id)
+	require.NoError(t, err)
+	any, ok := c.Extensions["io.cri-containerd.sandbox.metadata"]
+	require.True(t, ok, "sandbox metadata should exist in extension")
+	i, err := typeurl.UnmarshalAny(any)
+	require.NoError(t, err)
+	require.IsType(t, &sandbox.Metadata{}, i)
+	metadata, ok := i.(*sandbox.Metadata)
+	require.True(t, ok)
+	assert.NotEmpty(t, metadata.NetNSPath)
+	assert.Equal(t, netNS, metadata.NetNSPath, "network namespace path should be the same in runtime spec and sandbox metadata")
+}
+
+func ensureCNIAddRunning(t *testing.T, sbName string) error {
+	return Eventually(func() (bool, error) {
+		pids, err := PidsOf(failpointCNIBinary)
+		if err != nil || len(pids) == 0 {
+			return false, err
+		}
+
+		for _, pid := range pids {
+			envs, err := PidEnvs(pid)
+			if err != nil {
+				t.Logf("failed to read environ of pid %v: %v: skip it", pid, err)
+				continue
+			}
+
+			args, ok := envs["CNI_ARGS"]
+			if !ok {
+				t.Logf("expected CNI_ARGS env but got nothing, skip pid=%v", pid)
+				continue
+			}
+
+			for _, arg := range strings.Split(args, ";") {
+				kv := strings.SplitN(arg, "=", 2)
+				if len(kv) != 2 {
+					continue
+				}
+
+				if kv[0] == "K8S_POD_NAME" && kv[1] == sbName {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	}, time.Second, 30*time.Second)
+}
+
 // failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
 type failpointConf struct {
 	Add   string `json:"cmdAdd"`