diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4396cb003..6b9cefe0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -425,6 +425,10 @@ jobs: sudo apt-get update sudo apt-get install -y criu + - name: Install failpoint binaries + run: | + script/setup/install-failpoint-binaries + - name: Install containerd env: CGO_ENABLED: 1 diff --git a/Makefile b/Makefile index 648766c94..ecf166ac4 100644 --- a/Makefile +++ b/Makefile @@ -223,6 +223,16 @@ cri-integration: binaries bin/cri-integration.test ## run cri integration tests @bash -x ./script/test/cri-integration.sh @rm -rf bin/cri-integration.test +# build runc shimv2 with failpoint control, only used by integration test +bin/containerd-shim-runc-fp-v1: integration/failpoint/cmd/containerd-shim-runc-fp-v1 FORCE + @echo "$(WHALE) $@" + @CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./integration/failpoint/cmd/containerd-shim-runc-fp-v1 + +# build CNI bridge plugin wrapper with failpoint support, only used by integration test +bin/cni-bridge-fp: integration/failpoint/cmd/cni-bridge-fp FORCE + @echo "$(WHALE) $@" + @$(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/cni-bridge-fp + benchmark: ## run benchmarks tests @echo "$(WHALE) $@" @$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root @@ -374,6 +384,8 @@ clean-test: ## clean up debris from previously failed tests @rm -rf /run/containerd/fifo/* @rm -rf /run/containerd-test/* @rm -rf bin/cri-integration.test + @rm -rf bin/cni-bridge-fp + @rm -rf bin/containerd-shim-runc-fp-v1 install: ## install binaries @echo "$(WHALE) $@ $(BINARIES)" diff --git a/cmd/ctr/commands/commands.go b/cmd/ctr/commands/commands.go index 37ed75685..4b8b9cf20 100644 --- a/cmd/ctr/commands/commands.go +++ b/cmd/ctr/commands/commands.go @@ -116,6 +116,10 @@ var ( Name: "label", Usage: "specify additional labels (e.g. foo=bar)", }, + cli.StringSliceFlag{ + Name: "annotation", + Usage: "specify additional OCI annotations (e.g. foo=bar)", + }, cli.StringSliceFlag{ Name: "mount", Usage: "specify additional container mount (e.g. type=bind,src=/tmp,dst=/host,options=rbind:ro)", @@ -239,6 +243,19 @@ func LabelArgs(labelStrings []string) map[string]string { return labels } +// AnnotationArgs returns a map of annotation key,value pairs. +func AnnotationArgs(annoStrings []string) (map[string]string, error) { + annotations := make(map[string]string, len(annoStrings)) + for _, anno := range annoStrings { + parts := strings.SplitN(anno, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid key=value format annotation: %v", anno) + } + annotations[parts[0]] = parts[1] + } + return annotations, nil +} + // PrintAsJSON prints input in JSON format func PrintAsJSON(x interface{}) { b, err := json.MarshalIndent(x, "", " ") diff --git a/cmd/ctr/commands/run/run_unix.go b/cmd/ctr/commands/run/run_unix.go index 0ac5fe409..58acc2201 100644 --- a/cmd/ctr/commands/run/run_unix.go +++ b/cmd/ctr/commands/run/run_unix.go @@ -217,6 +217,13 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}), ) } + if annoStrings := context.StringSlice("annotation"); len(annoStrings) > 0 { + annos, err := commands.AnnotationArgs(annoStrings) + if err != nil { + return nil, err + } + opts = append(opts, oci.WithAnnotations(annos)) + } if context.Bool("cni") { cniMeta := &commands.NetworkMetaData{EnableCni: true} diff --git a/go.mod b/go.mod index b11b667df..457b754cb 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/containerd/ttrpc v1.1.1-0.20220420014843-944ef4a40df3 github.com/containerd/typeurl v1.0.3-0.20220422153119-7f6e6d160d67 github.com/containerd/zfs v1.0.0 + github.com/containernetworking/cni v1.1.1 github.com/containernetworking/plugins v1.1.1 github.com/coreos/go-systemd/v22 v22.3.2 github.com/davecgh/go-spew v1.1.1 @@ -82,7 +83,6 @@ require ( github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cilium/ebpf v0.7.0 // indirect - github.com/containernetworking/cni v1.1.1 // indirect github.com/containers/ocicrypt v1.1.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect diff --git a/integration/failpoint/cmd/cni-bridge-fp/README.md b/integration/failpoint/cmd/cni-bridge-fp/README.md new file mode 100644 index 000000000..065e3b5f0 --- /dev/null +++ b/integration/failpoint/cmd/cni-bridge-fp/README.md @@ -0,0 +1,159 @@ +## cni-bridge-f(ail)p(oint) + +### Overview + +The `cni-bridge-fp` is a CNI plugin which delegates interface-creating function +to [CNI bridge plugin][1] and allows user to inject failpoint before delegation. + +Since the CNI plugin is invoked by binary call from CRI and it is short-lived, +the failpoint need to be configured by a JSON file, which can be persisted. +There is an example about failpoint description. + +```json +{ + "cmdAdd": "1*error(you-shall-not-pass!)->1*panic(again)", + "cmdDel": "1*error(try-again)", + "cmdCheck": "10*off" +} +``` + +* `cmdAdd` (string, optional): The failpoint for `ADD` command. +* `cmdDel` (string, optional): The failpoint for `DEL` command. +* `cmdCheck` (string, optional): The failpoint for `CHECK` command. + +Since the `cmdXXX` can be multiple failpoints, each CNI binary call will update +the current state to make sure the order of execution is expected. + +And the failpoint injection is enabled by pod's annotation. Currently, the key +of customized CNI capabilities in containerd can only be `io.kubernetes.cri.pod-annotations` +and containerd will pass pod's annotations to CNI under the that object. The +user can use the `failpoint.cni.containerd.io/confpath` annotation to enable +failpoint for the pod. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx + annotations: + failpoint.cni.containerd.io/confpath: "/tmp/pod-failpoints.json" +spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +``` + +### Example + +Let's use the following json as failpoint description. + +```bash +$ cat <1*panic(sorry)" +} +EOF +``` + +And use `ip netns` to create persisted net namespace named by `failpoint`. + +```bash +$ sudo ip netns add failpoint +``` + +And then setup the following bash script for demo. + +```bash +$ cat <1*panic(sorry)" +} +``` + +We should setup CNI successfully after retry. When we teardown the interface, +there should be two failpoints. + +```bash +$ sudo CNI_COMMAND=ADD bash /tmp/cni-failpoint-demo-helper.sh +... + +$ sudo CNI_COMMAND=DEL bash /tmp/cni-failpoint-demo-helper.sh +{ + "code": 999, + "msg": "oops" +} + +$ sudo CNI_COMMAND=DEL bash /tmp/cni-failpoint-demo-helper.sh +{ + "code": 999, + "msg": "oops" +} + +$ cat /tmp/cni-failpoint.json | jq . +{ + "cmdAdd": "0*error(try-again)", + "cmdDel": "0*error(oops)", + "cmdCheck": "1*off->1*panic(sorry)" +} +``` + +[1]: diff --git a/integration/failpoint/cmd/cni-bridge-fp/main_linux.go b/integration/failpoint/cmd/cni-bridge-fp/main_linux.go new file mode 100644 index 000000000..125c8cf1e --- /dev/null +++ b/integration/failpoint/cmd/cni-bridge-fp/main_linux.go @@ -0,0 +1,202 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "syscall" + + "github.com/containerd/containerd/pkg/failpoint" + "github.com/containerd/continuity" + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" +) + +const delegatedPlugin = "bridge" + +type netConf struct { + RuntimeConfig struct { + PodAnnotations inheritedPodAnnotations `json:"io.kubernetes.cri.pod-annotations"` + } `json:"runtimeConfig,omitempty"` +} + +type inheritedPodAnnotations struct { + // FailpointConfPath represents filepath of failpoint settings. + FailpointConfPath string `json:"failpoint.cni.containerd.io/confpath,omitempty"` +} + +// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. +type failpointConf struct { + Add string `json:"cmdAdd,omitempty"` + Del string `json:"cmdDel,omitempty"` + Check string `json:"cmdCheck,omitempty"` +} + +func main() { + skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, "bridge with failpoint support") +} + +func cmdAdd(args *skel.CmdArgs) error { + if err := handleFailpoint(args, "ADD"); err != nil { + return err + } + + result, err := invoke.DelegateAdd(context.TODO(), delegatedPlugin, args.StdinData, nil) + if err != nil { + return err + } + return result.Print() +} + +func cmdCheck(args *skel.CmdArgs) error { + if err := handleFailpoint(args, "CHECK"); err != nil { + return err + } + + return invoke.DelegateCheck(context.TODO(), delegatedPlugin, args.StdinData, nil) +} + +func cmdDel(args *skel.CmdArgs) error { + if err := handleFailpoint(args, "DEL"); err != nil { + return err + } + + return invoke.DelegateDel(context.TODO(), delegatedPlugin, args.StdinData, nil) +} + +func handleFailpoint(args *skel.CmdArgs, cmdKind string) error { + var conf netConf + if err := json.Unmarshal(args.StdinData, &conf); err != nil { + return fmt.Errorf("failed to parse network configuration: %w", err) + } + + confPath := conf.RuntimeConfig.PodAnnotations.FailpointConfPath + if len(confPath) == 0 { + return nil + } + + control, err := newFailpointControl(confPath) + if err != nil { + return err + } + + evalFn, err := control.delegatedEvalFn(cmdKind) + if err != nil { + return err + } + return evalFn() +} + +type failpointControl struct { + confPath string +} + +func newFailpointControl(confPath string) (*failpointControl, error) { + if !filepath.IsAbs(confPath) { + return nil, fmt.Errorf("failpoint confPath(%s) is required to be absolute", confPath) + } + + return &failpointControl{ + confPath: confPath, + }, nil +} + +func (c *failpointControl) delegatedEvalFn(cmdKind string) (failpoint.EvalFn, error) { + var resFn failpoint.EvalFn = nopEvalFn + + if err := c.updateTx(func(conf *failpointConf) error { + var fpStr *string + + switch cmdKind { + case "ADD": + fpStr = &conf.Add + case "DEL": + fpStr = &conf.Del + case "CHECK": + fpStr = &conf.Check + } + + if fpStr == nil || *fpStr == "" { + return nil + } + + fp, err := failpoint.NewFailpoint(cmdKind, *fpStr) + if err != nil { + return fmt.Errorf("failed to parse failpoint %s: %w", *fpStr, err) + } + + resFn = fp.DelegatedEval() + + *fpStr = fp.Marshal() + return nil + + }); err != nil { + return nil, err + } + return resFn, nil +} + +func (c *failpointControl) updateTx(updateFn func(conf *failpointConf) error) error { + f, err := os.OpenFile(c.confPath, os.O_RDWR, 0666) + if err != nil { + return fmt.Errorf("failed to open confPath %s: %w", c.confPath, err) + } + defer f.Close() + + if err := flock(f.Fd()); err != nil { + return fmt.Errorf("failed to lock failpoint setting %s: %w", c.confPath, err) + } + defer unflock(f.Fd()) + + data, err := ioutil.ReadAll(f) + if err != nil { + return fmt.Errorf("failed to read failpoint setting %s: %w", c.confPath, err) + } + + var conf failpointConf + if err := json.Unmarshal(data, &conf); err != nil { + return fmt.Errorf("failed to unmarshal failpoint conf %s: %w", string(data), err) + } + + if err := updateFn(&conf); err != nil { + return err + } + + data, err = json.Marshal(conf) + if err != nil { + return fmt.Errorf("failed to marshal failpoint conf: %w", err) + } + return continuity.AtomicWriteFile(c.confPath, data, 0666) +} + +func nopEvalFn() error { + return nil +} + +func flock(fd uintptr) error { + return syscall.Flock(int(fd), syscall.LOCK_EX) +} + +func unflock(fd uintptr) error { + return syscall.Flock(int(fd), syscall.LOCK_UN) +} diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main_linux.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main_linux.go new file mode 100644 index 000000000..0c6af2526 --- /dev/null +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main_linux.go @@ -0,0 +1,29 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + + "github.com/containerd/containerd/runtime/v2/runc/manager" + _ "github.com/containerd/containerd/runtime/v2/runc/pause" + "github.com/containerd/containerd/runtime/v2/shim" +) + +func main() { + shim.RunManager(context.Background(), manager.NewShimManager("io.containerd.runc-fp.v1")) +} diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin_linux.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin_linux.go new file mode 100644 index 000000000..1e70c3c2a --- /dev/null +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin_linux.go @@ -0,0 +1,141 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + taskapi "github.com/containerd/containerd/api/runtime/task/v2" + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/failpoint" + "github.com/containerd/containerd/pkg/shutdown" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/runtime/v2/runc/task" + "github.com/containerd/containerd/runtime/v2/shim" + "github.com/containerd/ttrpc" +) + +const ( + ociConfigFilename = "config.json" + + failpointPrefixKey = "io.containerd.runtime.v2.shim.failpoint." +) + +func init() { + plugin.Register(&plugin.Registration{ + Type: plugin.TTRPCPlugin, + ID: "task", + Requires: []plugin.Type{ + plugin.EventPlugin, + plugin.InternalPlugin, + }, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + pp, err := ic.GetByID(plugin.EventPlugin, "publisher") + if err != nil { + return nil, err + } + ss, err := ic.GetByID(plugin.InternalPlugin, "shutdown") + if err != nil { + return nil, err + } + fps, err := newFailpointFromOCIAnnotation() + if err != nil { + return nil, err + } + service, err := task.NewTaskService(ic.Context, pp.(shim.Publisher), ss.(shutdown.Service)) + if err != nil { + return nil, err + } + + return &taskServiceWithFp{ + fps: fps, + local: service, + }, nil + }, + }) + +} + +type taskServiceWithFp struct { + fps map[string]*failpoint.Failpoint + local taskapi.TaskService +} + +func (s *taskServiceWithFp) RegisterTTRPC(server *ttrpc.Server) error { + taskapi.RegisterTaskService(server, s.local) + return nil +} + +func (s *taskServiceWithFp) UnaryInterceptor() ttrpc.UnaryServerInterceptor { + return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + methodName := filepath.Base(info.FullMethod) + if fp, ok := s.fps[methodName]; ok { + if err := fp.Evaluate(); err != nil { + return nil, err + } + } + return method(ctx, unmarshal) + } +} + +// newFailpointFromOCIAnnotation reloads and parses the annotation from +// bundle-path/config.json. +// +// The annotation controlling task API's failpoint should be like: +// +// io.containerd.runtime.v2.shim.failpoint.Create = 1*off->1*error(please retry) +// +// The `Create` is the shim unary API and the value of annotation is the +// failpoint control. The function will return a set of failpoint controllers. +func newFailpointFromOCIAnnotation() (map[string]*failpoint.Failpoint, error) { + // NOTE: shim's current working dir is in bundle dir. + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current working dir: %w", err) + } + + configPath := filepath.Join(cwd, ociConfigFilename) + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %w", configPath, err) + } + + var spec oci.Spec + if err := json.Unmarshal(data, &spec); err != nil { + return nil, fmt.Errorf("failed to parse oci.Spec(%v): %w", string(data), err) + } + + res := make(map[string]*failpoint.Failpoint) + for k, v := range spec.Annotations { + if !strings.HasPrefix(k, failpointPrefixKey) { + continue + } + + methodName := strings.TrimPrefix(k, failpointPrefixKey) + fp, err := failpoint.NewFailpoint(methodName, v) + if err != nil { + return nil, fmt.Errorf("failed to parse failpoint %v: %w", v, err) + } + res[methodName] = fp + } + return res, nil +} diff --git a/integration/main_test.go b/integration/main_test.go index 7befa4a9b..73c607eae 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -174,7 +174,8 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb Uid: util.GenerateID(), Namespace: Randomize(ns), }, - Linux: &runtime.LinuxPodSandboxConfig{}, + Linux: &runtime.LinuxPodSandboxConfig{}, + Annotations: make(map[string]string), } for _, opt := range opts { opt(config) diff --git a/integration/sandbox_run_rollback_test.go b/integration/sandbox_run_rollback_test.go new file mode 100644 index 000000000..4ab5829b6 --- /dev/null +++ b/integration/sandbox_run_rollback_test.go @@ -0,0 +1,122 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/pkg/failpoint" + "github.com/stretchr/testify/require" +) + +const ( + failpointRuntimeHandler = "runc-fp" + + failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint." + + failpointCNIConfPathKey = "failpoint.cni.containerd.io/confpath" +) + +func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + + t.Logf("Inject CNI failpoint") + conf := &failpointConf{ + Add: "1*error(you-shall-not-pass!)", + } + + sbConfig := PodSandboxConfig(t.Name(), "failpoint") + injectCNIFailpoint(t, sbConfig, conf) + + t.Logf("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Equal(t, true, strings.Contains(err.Error(), "you-shall-not-pass!")) + + t.Logf("Retry to create sandbox with same config") + sb, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.NoError(t, err) + + err = runtimeService.StopPodSandbox(sb) + require.NoError(t, err) + + err = runtimeService.RemovePodSandbox(sb) + require.NoError(t, err) +} + +func TestRunPodSandboxWithShimStartFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + + t.Logf("Inject Shim failpoint") + + sbConfig := PodSandboxConfig(t.Name(), "failpoint") + injectShimFailpoint(t, sbConfig, map[string]string{ + "Start": "1*error(no hard feelings)", + }) + + t.Logf("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings")) +} + +// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. +type failpointConf struct { + Add string `json:"cmdAdd"` + Del string `json:"cmdDel"` + Check string `json:"cmdCheck"` +} + +func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf *failpointConf) { + stateDir := t.TempDir() + + metadata := sbConfig.Metadata + fpFilename := filepath.Join(stateDir, + fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name)) + + data, err := json.Marshal(conf) + require.NoError(t, err) + + err = os.WriteFile(fpFilename, data, 0666) + require.NoError(t, err) + + sbConfig.Annotations[failpointCNIConfPathKey] = fpFilename +} + +func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) { + for method, fp := range methodFps { + _, err := failpoint.NewFailpoint(method, fp) + require.NoError(t, err, "check failpoint %s for shim method %s", fp, method) + + sbConfig.Annotations[failpointShimPrefixKey+method] = fp + } +} diff --git a/pkg/failpoint/fail.go b/pkg/failpoint/fail.go new file mode 100644 index 000000000..d48d65ba8 --- /dev/null +++ b/pkg/failpoint/fail.go @@ -0,0 +1,310 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package failpoint provides the code point in the path, which can be controlled +// by user's variable. +// +// Inspired by FreeBSD fail(9): https://freebsd.org/cgi/man.cgi?query=fail. +package failpoint + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "sync" + "time" +) + +// EvalFn is the func type about delegated evaluation. +type EvalFn func() error + +// Type is the type of failpoint to specifies which action to take. +type Type int + +const ( + // TypeInvalid is invalid type + TypeInvalid Type = iota + // TypeOff takes no action + TypeOff + // TypeError triggers failpoint error with specified argument + TypeError + // TypePanic triggers panic with specified argument + TypePanic + // TypeDelay sleeps with the specified number of milliseconds + TypeDelay +) + +// String returns the name of type. +func (t Type) String() string { + switch t { + case TypeOff: + return "off" + case TypeError: + return "error" + case TypePanic: + return "panic" + case TypeDelay: + return "delay" + default: + return "invalid" + } +} + +// Failpoint is used to add code points where error or panic may be injected by +// user. The user controlled variable will be parsed for how the error injected +// code should fire. There is the way to set the rule for failpoint. +// +// *[(arg)][->] +// +// The argument specifies which action to take; it can be one of: +// +// off: Takes no action (does not trigger failpoint and no argument) +// error: Triggers failpoint error with specified argument(string) +// panic: Triggers panic with specified argument(string) +// delay: Sleep the specified number of milliseconds +// +// The * modifiers prior to control when is executed. For +// example, "5*error(oops)" means "return error oops 5 times total". The +// operator -> can be used to express cascading terms. If you specify +// ->, it means that if does not execute, will +// be evaluated. If you want the error injected code should fire in second +// call, you can specify "1*off->1*error(oops)". +// +// Inspired by FreeBSD fail(9): https://freebsd.org/cgi/man.cgi?query=fail. +type Failpoint struct { + sync.Mutex + + fnName string + entries []*failpointEntry +} + +// NewFailpoint returns failpoint control. +func NewFailpoint(fnName string, terms string) (*Failpoint, error) { + entries, err := parseTerms([]byte(terms)) + if err != nil { + return nil, err + } + + return &Failpoint{ + fnName: fnName, + entries: entries, + }, nil +} + +// Evaluate evaluates a failpoint. +func (fp *Failpoint) Evaluate() error { + fn := fp.DelegatedEval() + return fn() +} + +// DelegatedEval evaluates a failpoint but delegates to caller to fire that. +func (fp *Failpoint) DelegatedEval() EvalFn { + var target *failpointEntry + + func() { + fp.Lock() + defer fp.Unlock() + + for _, entry := range fp.entries { + if entry.count == 0 { + continue + } + + entry.count-- + target = entry + break + } + }() + + if target == nil { + return nopEvalFn + } + return target.evaluate +} + +// Failpoint returns the current state of control in string format. +func (fp *Failpoint) Marshal() string { + fp.Lock() + defer fp.Unlock() + + res := make([]string, 0, len(fp.entries)) + for _, entry := range fp.entries { + res = append(res, entry.marshal()) + } + return strings.Join(res, "->") +} + +type failpointEntry struct { + typ Type + arg interface{} + count int64 +} + +func newFailpointEntry() *failpointEntry { + return &failpointEntry{ + typ: TypeInvalid, + count: 0, + } +} + +func (fpe *failpointEntry) marshal() string { + base := fmt.Sprintf("%d*%s", fpe.count, fpe.typ) + switch fpe.typ { + case TypeOff: + return base + case TypeError, TypePanic: + return fmt.Sprintf("%s(%s)", base, fpe.arg.(string)) + case TypeDelay: + return fmt.Sprintf("%s(%d)", base, fpe.arg.(time.Duration)/time.Millisecond) + default: + return base + } +} + +func (fpe *failpointEntry) evaluate() error { + switch fpe.typ { + case TypeOff: + return nil + case TypeError: + return fmt.Errorf("%v", fpe.arg) + case TypePanic: + panic(fpe.arg) + case TypeDelay: + time.Sleep(fpe.arg.(time.Duration)) + return nil + default: + panic("invalid failpoint type") + } +} + +func parseTerms(term []byte) ([]*failpointEntry, error) { + var entry *failpointEntry + var err error + + // count*type[(arg)] + term, entry, err = parseTerm(term) + if err != nil { + return nil, err + } + + res := []*failpointEntry{entry} + + // cascading terms + for len(term) > 0 { + if !bytes.HasPrefix(term, []byte("->")) { + return nil, fmt.Errorf("invalid cascading terms: %s", string(term)) + } + + term = term[2:] + term, entry, err = parseTerm(term) + if err != nil { + return nil, fmt.Errorf("failed to parse cascading term: %w", err) + } + + res = append(res, entry) + } + return res, nil +} + +func parseTerm(term []byte) ([]byte, *failpointEntry, error) { + var err error + var entry = newFailpointEntry() + + // count* + term, err = parseInt64(term, '*', &entry.count) + if err != nil { + return nil, nil, err + } + + // type[(arg)] + term, err = parseType(term, entry) + return term, entry, err +} + +func parseType(term []byte, entry *failpointEntry) ([]byte, error) { + var nameToTyp = map[string]Type{ + "off": TypeOff, + "error(": TypeError, + "panic(": TypePanic, + "delay(": TypeDelay, + } + + var found bool + for name, typ := range nameToTyp { + if bytes.HasPrefix(term, []byte(name)) { + found = true + term = term[len(name):] + entry.typ = typ + break + } + } + + if !found { + return nil, fmt.Errorf("invalid type format: %s", string(term)) + } + + switch entry.typ { + case TypePanic, TypeError: + endIdx := bytes.IndexByte(term, ')') + if endIdx <= 0 { + return nil, fmt.Errorf("invalid argument for %s type", entry.typ) + } + entry.arg = string(term[:endIdx]) + return term[endIdx+1:], nil + case TypeOff: + // do nothing + return term, nil + case TypeDelay: + var msVal int64 + var err error + + term, err = parseInt64(term, ')', &msVal) + if err != nil { + return nil, err + } + entry.arg = time.Millisecond * time.Duration(msVal) + return term, nil + default: + panic("unreachable") + } +} + +func parseInt64(term []byte, terminate byte, val *int64) ([]byte, error) { + i := 0 + + for ; i < len(term); i++ { + if b := term[i]; b < '0' || b > '9' { + break + } + } + + if i == 0 || i == len(term) || term[i] != terminate { + return nil, fmt.Errorf("failed to parse int64 because of invalid terminate byte: %s", string(term)) + } + + v, err := strconv.ParseInt(string(term[:i]), 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse int64 from %s: %v", string(term[:i]), err) + } + + *val = v + return term[i+1:], nil +} + +func nopEvalFn() error { + return nil +} diff --git a/pkg/failpoint/fail_test.go b/pkg/failpoint/fail_test.go new file mode 100644 index 000000000..1b79ff6eb --- /dev/null +++ b/pkg/failpoint/fail_test.go @@ -0,0 +1,134 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package failpoint + +import ( + "reflect" + "testing" + "time" +) + +func TestParseTerms(t *testing.T) { + cases := []struct { + terms string + hasError bool + }{ + // off + {"5", true}, + {"*off()", true}, + {"5*off()", true}, + {"5*off(nothing)", true}, + {"5*off(", true}, + {"5*off", false}, + + // error + {"10000error(oops)", true}, + {"10*error(oops)", false}, + {"1234*error(oops))", true}, + {"12342*error()", true}, + + // panic + {"1panic(oops)", true}, + {"1000000*panic(oops)", false}, + {"12345*panic(oops))", true}, + {"12*panic()", true}, + + // delay + {"1*delay(oops)", true}, + {"1000000*delay(-1)", true}, + {"1000000*delay(1)", false}, + + // cascading terms + {"1*delay(1)-", true}, + {"10*delay(2)->", true}, + {"11*delay(3)->10*off(", true}, + {"12*delay(4)->10*of", true}, + {"13*delay(5)->10*off->1000*panic(oops)", false}, + } + + for i, c := range cases { + fp, err := NewFailpoint(t.Name(), c.terms) + + if (err != nil && !c.hasError) || + (err == nil && c.hasError) { + + t.Fatalf("[%v - %s] expected hasError=%v, but got %v", i, c.terms, c.hasError, err) + } + + if err != nil { + continue + } + + if got := fp.Marshal(); !reflect.DeepEqual(got, c.terms) { + t.Fatalf("[%v] expected %v, but got %v", i, c.terms, got) + } + } +} + +func TestEvaluate(t *testing.T) { + terms := "1*error(oops-)->1*off->1*delay(1000)->1*panic(panic)" + + fp, err := NewFailpoint(t.Name(), terms) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + injectedFn := func() error { + if err := fp.Evaluate(); err != nil { + return err + } + return nil + } + + // should return oops- error + if err := injectedFn(); err == nil || err.Error() != "oops-" { + t.Fatalf("expected error %v, but got %v", "oops-", err) + } + + // should return nil + if err := injectedFn(); err != nil { + t.Fatalf("expected nil, but got %v", err) + } + + // should sleep 1s and return nil + now := time.Now() + err = injectedFn() + du := time.Since(now) + if err != nil { + t.Fatalf("expected nil, but got %v", err) + } + if du < 1*time.Second { + t.Fatalf("expected sleep 1s, but got %v", du) + } + + // should panic + defer func() { + if err := recover(); err == nil || err.(string) != "panic" { + t.Fatalf("should panic(panic), but got %v", err) + } + + expected := "0*error(oops-)->0*off->0*delay(1000)->0*panic(panic)" + if got := fp.Marshal(); got != expected { + t.Fatalf("expected %v, but got %v", expected, got) + } + + if err := injectedFn(); err != nil { + t.Fatalf("expected nil, but got %v", err) + } + }() + injectedFn() +} diff --git a/runtime/v2/shim/shim.go b/runtime/v2/shim/shim.go index 9fa319367..a6078a569 100644 --- a/runtime/v2/shim/shim.go +++ b/runtime/v2/shim/shim.go @@ -108,6 +108,12 @@ type ttrpcService interface { RegisterTTRPC(*ttrpc.Server) error } +type ttrpcServerOptioner interface { + ttrpcService + + UnaryInterceptor() ttrpc.UnaryServerInterceptor +} + type taskService struct { shimapi.TaskService } @@ -370,6 +376,8 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi var ( initialized = plugin.NewPluginSet() ttrpcServices = []ttrpcService{} + + ttrpcUnaryInterceptors = []ttrpc.UnaryServerInterceptor{} ) plugins := plugin.Graph(func(*plugin.Registration) bool { return false }) for _, p := range plugins { @@ -409,19 +417,28 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi if err != nil { if plugin.IsSkipPlugin(err) { log.G(ctx).WithError(err).WithField("type", p.Type).Infof("skip loading plugin %q...", id) - } else { - log.G(ctx).WithError(err).Warnf("failed to load plugin %s", id) + continue } - continue + return fmt.Errorf("failed to load plugin %s: %w", id, err) } if src, ok := instance.(ttrpcService); ok { logrus.WithField("id", id).Debug("registering ttrpc service") ttrpcServices = append(ttrpcServices, src) + + } + + if src, ok := instance.(ttrpcServerOptioner); ok { + ttrpcUnaryInterceptors = append(ttrpcUnaryInterceptors, src.UnaryInterceptor()) } } - server, err := newServer() + if len(ttrpcServices) == 0 { + return fmt.Errorf("required that ttrpc service") + } + + unaryInterceptor := chainUnaryServerInterceptors(ttrpcUnaryInterceptors...) + server, err := newServer(ttrpc.WithUnaryServerInterceptor(unaryInterceptor)) if err != nil { return fmt.Errorf("failed creating server: %w", err) } diff --git a/runtime/v2/shim/shim_darwin.go b/runtime/v2/shim/shim_darwin.go index fe833df01..0bdf289bb 100644 --- a/runtime/v2/shim/shim_darwin.go +++ b/runtime/v2/shim/shim_darwin.go @@ -18,8 +18,8 @@ package shim import "github.com/containerd/ttrpc" -func newServer() (*ttrpc.Server, error) { - return ttrpc.NewServer() +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { + return ttrpc.NewServer(opts...) } func subreaper() error { diff --git a/runtime/v2/shim/shim_freebsd.go b/runtime/v2/shim/shim_freebsd.go index fe833df01..0bdf289bb 100644 --- a/runtime/v2/shim/shim_freebsd.go +++ b/runtime/v2/shim/shim_freebsd.go @@ -18,8 +18,8 @@ package shim import "github.com/containerd/ttrpc" -func newServer() (*ttrpc.Server, error) { - return ttrpc.NewServer() +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { + return ttrpc.NewServer(opts...) } func subreaper() error { diff --git a/runtime/v2/shim/shim_linux.go b/runtime/v2/shim/shim_linux.go index 06266a533..1c05c2c56 100644 --- a/runtime/v2/shim/shim_linux.go +++ b/runtime/v2/shim/shim_linux.go @@ -21,8 +21,9 @@ import ( "github.com/containerd/ttrpc" ) -func newServer() (*ttrpc.Server, error) { - return ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser())) +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { + opts = append(opts, ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser())) + return ttrpc.NewServer(opts...) } func subreaper() error { diff --git a/runtime/v2/shim/shim_windows.go b/runtime/v2/shim/shim_windows.go index 4b098ab16..2add7ac33 100644 --- a/runtime/v2/shim/shim_windows.go +++ b/runtime/v2/shim/shim_windows.go @@ -31,7 +31,7 @@ func setupSignals(config Config) (chan os.Signal, error) { return nil, errors.New("not supported") } -func newServer() (*ttrpc.Server, error) { +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { return nil, errors.New("not supported") } diff --git a/runtime/v2/shim/util.go b/runtime/v2/shim/util.go index bf78fa58f..d1cd47946 100644 --- a/runtime/v2/shim/util.go +++ b/runtime/v2/shim/util.go @@ -30,6 +30,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/protobuf/proto" "github.com/containerd/containerd/protobuf/types" + "github.com/containerd/ttrpc" exec "golang.org/x/sys/execabs" ) @@ -167,3 +168,28 @@ func ReadAddress(path string) (string, error) { } return string(data), nil } + +// chainUnaryServerInterceptors creates a single ttrpc server interceptor from +// a chain of many interceptors executed from first to last. +func chainUnaryServerInterceptors(interceptors ...ttrpc.UnaryServerInterceptor) ttrpc.UnaryServerInterceptor { + n := len(interceptors) + + // force to use default interceptor in ttrpc + if n == 0 { + return nil + } + + return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + currentMethod := method + + for i := n - 1; i > 0; i-- { + interceptor := interceptors[i] + innerMethod := currentMethod + + currentMethod = func(currentCtx context.Context, currentUnmarshal func(interface{}) error) (interface{}, error) { + return interceptor(currentCtx, currentUnmarshal, info, innerMethod) + } + } + return interceptors[0](ctx, unmarshal, info, currentMethod) + } +} diff --git a/runtime/v2/shim/util_test.go b/runtime/v2/shim/util_test.go new file mode 100644 index 000000000..8341bcddb --- /dev/null +++ b/runtime/v2/shim/util_test.go @@ -0,0 +1,118 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shim + +import ( + "context" + "path/filepath" + "reflect" + "testing" + + "github.com/containerd/ttrpc" +) + +func TestChainUnaryServerInterceptors(t *testing.T) { + methodInfo := &ttrpc.UnaryServerInfo{ + FullMethod: filepath.Join("/", t.Name(), "foo"), + } + + type callKey struct{} + callValue := "init" + callCtx := context.WithValue(context.Background(), callKey{}, callValue) + + verifyCallCtxFn := func(ctx context.Context, key interface{}, expected interface{}) { + got := ctx.Value(key) + if !reflect.DeepEqual(expected, got) { + t.Fatalf("[context(key:%s) expected %v, but got %v", key, expected, got) + } + } + + verifyInfoFn := func(info *ttrpc.UnaryServerInfo) { + if !reflect.DeepEqual(methodInfo, info) { + t.Fatalf("[info] expected %+v, but got %+v", methodInfo, info) + } + } + + origUnmarshaler := func(obj interface{}) error { + v := obj.(*int64) + *v *= 2 + return nil + } + + type firstKey struct{} + firstValue := "from first" + var firstUnmarshaler ttrpc.Unmarshaler + first := func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + verifyCallCtxFn(ctx, callKey{}, callValue) + verifyInfoFn(info) + + ctx = context.WithValue(ctx, firstKey{}, firstValue) + + firstUnmarshaler = func(obj interface{}) error { + if err := unmarshal(obj); err != nil { + return err + } + + v := obj.(*int64) + *v *= 2 + return nil + } + + return method(ctx, firstUnmarshaler) + } + + type secondKey struct{} + secondValue := "from second" + second := func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + verifyCallCtxFn(ctx, callKey{}, callValue) + verifyCallCtxFn(ctx, firstKey{}, firstValue) + verifyInfoFn(info) + + v := int64(3) // should return 12 + if err := unmarshal(&v); err != nil { + t.Fatalf("unexpected error %v", err) + } + if expected := int64(12); v != expected { + t.Fatalf("expected int64(%v), but got %v", expected, v) + } + + ctx = context.WithValue(ctx, secondKey{}, secondValue) + return method(ctx, unmarshal) + } + + methodFn := func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + verifyCallCtxFn(ctx, callKey{}, callValue) + verifyCallCtxFn(ctx, firstKey{}, firstValue) + verifyCallCtxFn(ctx, secondKey{}, secondValue) + + v := int64(2) + if err := unmarshal(&v); err != nil { + return nil, err + } + return v, nil + } + + interceptor := chainUnaryServerInterceptors(first, second) + v, err := interceptor(callCtx, origUnmarshaler, methodInfo, methodFn) + if err != nil { + t.Fatalf("expected nil, but got %v", err) + } + + if expected := int64(8); v != expected { + t.Fatalf("expected result is int64(%v), but got %v", expected, v) + } +} diff --git a/script/setup/install-failpoint-binaries b/script/setup/install-failpoint-binaries new file mode 100755 index 000000000..533eb5423 --- /dev/null +++ b/script/setup/install-failpoint-binaries @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build and install +# +# * cni-bridge-fp into /opt/cni/bin +# * containerd-shim-runc-fp-v1 into /usr/local/bin +# +set -euo pipefail + +base_dir="$(dirname "${BASH_SOURCE[0]}")" +root_dir="$( cd "${base_dir}" && pwd )"/../.. + +cd "${root_dir}" + +CNI_BIN_DIR=${CNI_BIN_DIR:-"/opt/cni/bin"} +make bin/cni-bridge-fp +sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}" + +SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"} +make bin/containerd-shim-runc-fp-v1 +sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}" diff --git a/script/test/utils.sh b/script/test/utils.sh index fa5e5b49d..595617813 100755 --- a/script/test/utils.sh +++ b/script/test/utils.sh @@ -35,9 +35,10 @@ CONTAINERD_RUNTIME=${CONTAINERD_RUNTIME:-""} if [ -z "${CONTAINERD_CONFIG_FILE}" ]; then config_file="${CONTAINERD_CONFIG_DIR}/containerd-config-cri.toml" truncate --size 0 "${config_file}" + echo "version=2" >> ${config_file} + if command -v sestatus >/dev/null 2>&1; then cat >>${config_file} < 0 { + joined := strings.Join(argsMissing, ",") + return "", nil, types.NewError(types.ErrInvalidEnvironmentVariables, fmt.Sprintf("required env variables [%s] missing", joined), "") + } + + if cmd == "VERSION" { + t.Stdin = bytes.NewReader(nil) + } + + stdinData, err := ioutil.ReadAll(t.Stdin) + if err != nil { + return "", nil, types.NewError(types.ErrIOFailure, fmt.Sprintf("error reading from stdin: %v", err), "") + } + + cmdArgs := &CmdArgs{ + ContainerID: contID, + Netns: netns, + IfName: ifName, + Args: args, + Path: path, + StdinData: stdinData, + } + return cmd, cmdArgs, nil +} + +func (t *dispatcher) checkVersionAndCall(cmdArgs *CmdArgs, pluginVersionInfo version.PluginInfo, toCall func(*CmdArgs) error) *types.Error { + configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData) + if err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } + verErr := t.VersionReconciler.Check(configVersion, pluginVersionInfo) + if verErr != nil { + return types.NewError(types.ErrIncompatibleCNIVersion, "incompatible CNI versions", verErr.Details()) + } + + if err = toCall(cmdArgs); err != nil { + if e, ok := err.(*types.Error); ok { + // don't wrap Error in Error + return e + } + return types.NewError(types.ErrInternal, err.Error(), "") + } + + return nil +} + +func validateConfig(jsonBytes []byte) *types.Error { + var conf struct { + Name string `json:"name"` + } + if err := json.Unmarshal(jsonBytes, &conf); err != nil { + return types.NewError(types.ErrDecodingFailure, fmt.Sprintf("error unmarshall network config: %v", err), "") + } + if conf.Name == "" { + return types.NewError(types.ErrInvalidNetworkConfig, "missing network name", "") + } + if err := utils.ValidateNetworkName(conf.Name); err != nil { + return err + } + return nil +} + +func (t *dispatcher) pluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) *types.Error { + cmd, cmdArgs, err := t.getCmdArgsFromEnv() + if err != nil { + // Print the about string to stderr when no command is set + if err.Code == types.ErrInvalidEnvironmentVariables && t.Getenv("CNI_COMMAND") == "" && about != "" { + _, _ = fmt.Fprintln(t.Stderr, about) + _, _ = fmt.Fprintf(t.Stderr, "CNI protocol versions supported: %s\n", strings.Join(versionInfo.SupportedVersions(), ", ")) + return nil + } + return err + } + + if cmd != "VERSION" { + if err = validateConfig(cmdArgs.StdinData); err != nil { + return err + } + if err = utils.ValidateContainerID(cmdArgs.ContainerID); err != nil { + return err + } + if err = utils.ValidateInterfaceName(cmdArgs.IfName); err != nil { + return err + } + } + + switch cmd { + case "ADD": + err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdAdd) + case "CHECK": + configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData) + if err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } + if gtet, err := version.GreaterThanOrEqualTo(configVersion, "0.4.0"); err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } else if !gtet { + return types.NewError(types.ErrIncompatibleCNIVersion, "config version does not allow CHECK", "") + } + for _, pluginVersion := range versionInfo.SupportedVersions() { + gtet, err := version.GreaterThanOrEqualTo(pluginVersion, configVersion) + if err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } else if gtet { + if err := t.checkVersionAndCall(cmdArgs, versionInfo, cmdCheck); err != nil { + return err + } + return nil + } + } + return types.NewError(types.ErrIncompatibleCNIVersion, "plugin version does not allow CHECK", "") + case "DEL": + err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdDel) + case "VERSION": + if err := versionInfo.Encode(t.Stdout); err != nil { + return types.NewError(types.ErrIOFailure, err.Error(), "") + } + default: + return types.NewError(types.ErrInvalidEnvironmentVariables, fmt.Sprintf("unknown CNI_COMMAND: %v", cmd), "") + } + + return err +} + +// PluginMainWithError is the core "main" for a plugin. It accepts +// callback functions for add, check, and del CNI commands and returns an error. +// +// The caller must also specify what CNI spec versions the plugin supports. +// +// It is the responsibility of the caller to check for non-nil error return. +// +// For a plugin to comply with the CNI spec, it must print any error to stdout +// as JSON and then exit with nonzero status code. +// +// To let this package automatically handle errors and call os.Exit(1) for you, +// use PluginMain() instead. +func PluginMainWithError(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) *types.Error { + return (&dispatcher{ + Getenv: os.Getenv, + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + }).pluginMain(cmdAdd, cmdCheck, cmdDel, versionInfo, about) +} + +// PluginMain is the core "main" for a plugin which includes automatic error handling. +// +// The caller must also specify what CNI spec versions the plugin supports. +// +// The caller can specify an "about" string, which is printed on stderr +// when no CNI_COMMAND is specified. The recommended output is "CNI plugin v" +// +// When an error occurs in either cmdAdd, cmdCheck, or cmdDel, PluginMain will print the error +// as JSON to stdout and call os.Exit(1). +// +// To have more control over error handling, use PluginMainWithError() instead. +func PluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) { + if e := PluginMainWithError(cmdAdd, cmdCheck, cmdDel, versionInfo, about); e != nil { + if err := e.Print(); err != nil { + log.Print("Error writing error JSON to stdout: ", err) + } + os.Exit(1) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 590b3d5f3..332b55d7d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -134,6 +134,7 @@ github.com/containerd/zfs/plugin ## explicit; go 1.14 github.com/containernetworking/cni/libcni github.com/containernetworking/cni/pkg/invoke +github.com/containernetworking/cni/pkg/skel github.com/containernetworking/cni/pkg/types github.com/containernetworking/cni/pkg/types/020 github.com/containernetworking/cni/pkg/types/040