From 3c5e80b63e683453fee48093347c7c1e969450a7 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 19 Jun 2022 21:48:10 +0800 Subject: [PATCH] integration: Add injected failpoint testing for RunPodSandbox Signed-off-by: Wei Fu --- .github/workflows/ci.yml | 4 + integration/main_test.go | 3 +- integration/sandbox_run_rollback_test.go | 122 +++++++++++++++++++++++ script/setup/install-failpoint-binaries | 35 +++++++ script/test/utils.sh | 58 ++++++++++- 5 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 integration/sandbox_run_rollback_test.go create mode 100755 script/setup/install-failpoint-binaries diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22258b2cc..f17b73665 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -425,6 +425,10 @@ jobs: sudo apt-get update sudo apt-get install -y criu + - name: Install failpoint binaries + run: | + script/setup/install-failpoint-binaries + - name: Install containerd env: CGO_ENABLED: 1 diff --git a/integration/main_test.go b/integration/main_test.go index 4659d0f70..18c054b8e 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -174,7 +174,8 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb Uid: util.GenerateID(), Namespace: Randomize(ns), }, - Linux: &runtime.LinuxPodSandboxConfig{}, + Linux: &runtime.LinuxPodSandboxConfig{}, + Annotations: make(map[string]string), } for _, opt := range opts { opt(config) diff --git a/integration/sandbox_run_rollback_test.go b/integration/sandbox_run_rollback_test.go new file mode 100644 index 000000000..fa4502ac8 --- /dev/null +++ b/integration/sandbox_run_rollback_test.go @@ -0,0 +1,122 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/pkg/failpoint" + "github.com/stretchr/testify/require" +) + +const ( + failpointRuntimeHandler = "runc-fp" + + failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint." + + failpointCNIStateDirKey = "cniFailpointControlStateDir" +) + +func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + + t.Logf("Inject CNI failpoint") + conf := &failpointConf{ + Add: "1*error(you-shall-not-pass!)", + } + + sbConfig := PodSandboxConfig(t.Name(), "failpoint") + injectCNIFailpoint(t, sbConfig, conf) + + t.Logf("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Equal(t, true, strings.Contains(err.Error(), "you-shall-not-pass!")) + + t.Logf("Retry to create sandbox with same config") + sb, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.NoError(t, err) + + err = runtimeService.StopPodSandbox(sb) + require.NoError(t, err) + + err = runtimeService.RemovePodSandbox(sb) + require.NoError(t, err) +} + +func TestRunPodSandboxWithShimStartFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + + t.Logf("Inject Shim failpoint") + + sbConfig := PodSandboxConfig(t.Name(), "failpoint") + injectShimFailpoint(t, sbConfig, map[string]string{ + "Start": "1*error(no hard feelings)", + }) + + t.Logf("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings")) +} + +// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. +type failpointConf struct { + Add string `json:"cmdAdd"` + Del string `json:"cmdDel"` + Check string `json:"cmdCheck"` +} + +func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf *failpointConf) { + stateDir := t.TempDir() + + metadata := sbConfig.Metadata + fpFilename := filepath.Join(stateDir, + fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name)) + + data, err := json.Marshal(conf) + require.NoError(t, err) + + err = os.WriteFile(fpFilename, data, 0666) + require.NoError(t, err) + + sbConfig.Annotations[failpointCNIStateDirKey] = stateDir +} + +func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) { + for method, fp := range methodFps { + _, err := failpoint.NewFailpoint(method, fp) + require.NoError(t, err, "check failpoint %s for shim method %s", fp, method) + + sbConfig.Annotations[failpointShimPrefixKey+method] = fp + } +} diff --git a/script/setup/install-failpoint-binaries b/script/setup/install-failpoint-binaries new file mode 100755 index 000000000..533eb5423 --- /dev/null +++ b/script/setup/install-failpoint-binaries @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build and install +# +# * cni-bridge-fp into /opt/cni/bin +# * containerd-shim-runc-fp-v1 into /usr/local/bin +# +set -euo pipefail + +base_dir="$(dirname "${BASH_SOURCE[0]}")" +root_dir="$( cd "${base_dir}" && pwd )"/../.. + +cd "${root_dir}" + +CNI_BIN_DIR=${CNI_BIN_DIR:-"/opt/cni/bin"} +make bin/cni-bridge-fp +sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}" + +SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"} +make bin/containerd-shim-runc-fp-v1 +sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}" diff --git a/script/test/utils.sh b/script/test/utils.sh index fa5e5b49d..595617813 100755 --- a/script/test/utils.sh +++ b/script/test/utils.sh @@ -35,9 +35,10 @@ CONTAINERD_RUNTIME=${CONTAINERD_RUNTIME:-""} if [ -z "${CONTAINERD_CONFIG_FILE}" ]; then config_file="${CONTAINERD_CONFIG_DIR}/containerd-config-cri.toml" truncate --size 0 "${config_file}" + echo "version=2" >> ${config_file} + if command -v sestatus >/dev/null 2>&1; then cat >>${config_file} <