github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #7069 from fuweid/failpoint-in-runc-shimv2

Browse Source 
test: introduce failpoint control to runc-shimv2 and cni
This commit is contained in:
Derek McGowan
2022-07-26 23:12:20 -07:00
committed by GitHub
parent f81b0e2057 e6a2c07902
commit 6acde90772
24 changed files with 1700 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View File

@@ -425,6 +425,10 @@ jobs:
sudo apt-get update
sudo apt-get install -y criu
- name: Install failpoint binaries
run: |
script/setup/install-failpoint-binaries
- name: Install containerd
env:
CGO_ENABLED: 1

12
Makefile
View File

@@ -223,6 +223,16 @@ cri-integration: binaries bin/cri-integration.test ## run cri integration tests
@bash -x ./script/test/cri-integration.sh
@rm -rf bin/cri-integration.test
# build runc shimv2 with failpoint control, only used by integration test
bin/containerd-shim-runc-fp-v1: integration/failpoint/cmd/containerd-shim-runc-fp-v1 FORCE
@echo "$(WHALE) $@"
@CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./integration/failpoint/cmd/containerd-shim-runc-fp-v1
# build CNI bridge plugin wrapper with failpoint support, only used by integration test
bin/cni-bridge-fp: integration/failpoint/cmd/cni-bridge-fp FORCE
@echo "$(WHALE) $@"
@$(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/cni-bridge-fp
benchmark: ## run benchmarks tests
@echo "$(WHALE) $@"
@$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root
@@ -374,6 +384,8 @@ clean-test: ## clean up debris from previously failed tests
@rm -rf /run/containerd/fifo/*
@rm -rf /run/containerd-test/*
@rm -rf bin/cri-integration.test
@rm -rf bin/cni-bridge-fp
@rm -rf bin/containerd-shim-runc-fp-v1
install: ## install binaries
@echo "$(WHALE) $@ $(BINARIES)"

17
cmd/ctr/commands/commands.go
View File

@@ -116,6 +116,10 @@ var (
Name: "label",
Usage: "specify additional labels (e.g. foo=bar)",
},
cli.StringSliceFlag{
Name: "annotation",
Usage: "specify additional OCI annotations (e.g. foo=bar)",
},
cli.StringSliceFlag{
Name: "mount",
Usage: "specify additional container mount (e.g. type=bind,src=/tmp,dst=/host,options=rbind:ro)",
@@ -239,6 +243,19 @@ func LabelArgs(labelStrings []string) map[string]string {
return labels
}
// AnnotationArgs returns a map of annotation key,value pairs.
func AnnotationArgs(annoStrings []string) (map[string]string, error) {
annotations := make(map[string]string, len(annoStrings))
for _, anno := range annoStrings {
parts := strings.SplitN(anno, "=", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid key=value format annotation: %v", anno)
}
annotations[parts[0]] = parts[1]
}
return annotations, nil
}
// PrintAsJSON prints input in JSON format
func PrintAsJSON(x interface{}) {
b, err := json.MarshalIndent(x, "", " ")

7
cmd/ctr/commands/run/run_unix.go
View File

@@ -217,6 +217,13 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli
oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}),
)
}
if annoStrings := context.StringSlice("annotation"); len(annoStrings) > 0 {
annos, err := commands.AnnotationArgs(annoStrings)
if err != nil {
return nil, err
}
opts = append(opts, oci.WithAnnotations(annos))
}
if context.Bool("cni") {
cniMeta := &commands.NetworkMetaData{EnableCni: true}

2
go.mod
View File

@@ -20,6 +20,7 @@ require (
github.com/containerd/ttrpc v1.1.1-0.20220420014843-944ef4a40df3
github.com/containerd/typeurl v1.0.3-0.20220422153119-7f6e6d160d67
github.com/containerd/zfs v1.0.0
github.com/containernetworking/cni v1.1.1
github.com/containernetworking/plugins v1.1.1
github.com/coreos/go-systemd/v22 v22.3.2
github.com/davecgh/go-spew v1.1.1
@@ -82,7 +83,6 @@ require (
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cilium/ebpf v0.7.0 // indirect
github.com/containernetworking/cni v1.1.1 // indirect
github.com/containers/ocicrypt v1.1.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect

159
integration/failpoint/cmd/cni-bridge-fp/README.md Normal file
View File

@@ -0,0 +1,159 @@
## cni-bridge-f(ail)p(oint)
### Overview
The `cni-bridge-fp` is a CNI plugin which delegates interface-creating function
to [CNI bridge plugin][1] and allows user to inject failpoint before delegation.
Since the CNI plugin is invoked by binary call from CRI and it is short-lived,
the failpoint need to be configured by a JSON file, which can be persisted.
There is an example about failpoint description.
```json
{
"cmdAdd": "1*error(you-shall-not-pass!)->1*panic(again)",
"cmdDel": "1*error(try-again)",
"cmdCheck": "10*off"
}
```
* `cmdAdd` (string, optional): The failpoint for `ADD` command.
* `cmdDel` (string, optional): The failpoint for `DEL` command.
* `cmdCheck` (string, optional): The failpoint for `CHECK` command.
Since the `cmdXXX` can be multiple failpoints, each CNI binary call will update
the current state to make sure the order of execution is expected.
And the failpoint injection is enabled by pod's annotation. Currently, the key
of customized CNI capabilities in containerd can only be `io.kubernetes.cri.pod-annotations`
and containerd will pass pod's annotations to CNI under the that object. The
user can use the `failpoint.cni.containerd.io/confpath` annotation to enable
failpoint for the pod.
```yaml
apiVersion: v1
kind: Pod
metadata:
name: nginx
annotations:
failpoint.cni.containerd.io/confpath: "/tmp/pod-failpoints.json"
spec:
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80
```
### Example
Let's use the following json as failpoint description.
```bash
$ cat <<EOF | tee /tmp/cni-failpoint.json
{
"cmdAdd": "1*error(try-again)",
"cmdDel": "2*error(oops)",
"cmdCheck": "1*off->1*panic(sorry)"
}
EOF
```
And use `ip netns` to create persisted net namespace named by `failpoint`.
```bash
$ sudo ip netns add failpoint
```
And then setup the following bash script for demo.
```bash
$ cat <<EOFDEMO | tee /tmp/cni-failpoint-demo-helper.sh
#!/usr/bin/env bash
export CNI_CONTAINERID=failpoint-testing
export CNI_NETNS=/run/netns/failpoint
export CNI_IFNAME=fpeni0
export CNI_PATH=/opt/cni/bin/
cat <<EOF | /opt/cni/bin/cni-bridge-fp
{
"cniVersion": "0.3.0",
"name": "containerd-net-fp",
"type": "cni-bridge-fp",
"bridge": "fp-cni0",
"isGateway": true,
"ipMasq": true,
"promiscMode": true,
"ipam": {
"type": "host-local",
"ranges": [
[{
"subnet": "10.88.0.0/16"
}],
[{
"subnet": "2001:4860:4860::/64"
}]
],
"routes": [
{ "dst": "0.0.0.0/0" },
{ "dst": "::/0" }
]
},
"runtimeConfig": {
"io.kubernetes.cri.pod-annotations": {
"failpoint.cni.containerd.io/confpath": "/tmp/cni-failpoint.json"
}
}
}
EOF
EOFDEMO
```
Let's try to setup CNI and we should get a error `try-again`.
```bash
$ sudo CNI_COMMAND=ADD bash /tmp/cni-failpoint-demo-helper.sh
{
"code": 999,
"msg": "try-again"
}
# there is no failpoint for ADD command.
$ cat /tmp/cni-failpoint.json | jq .
{
"cmdAdd": "0*error(try-again)",
"cmdDel": "2*error(oops)",
"cmdCheck": "1*off->1*panic(sorry)"
}
```
We should setup CNI successfully after retry. When we teardown the interface,
there should be two failpoints.
```bash
$ sudo CNI_COMMAND=ADD bash /tmp/cni-failpoint-demo-helper.sh
...
$ sudo CNI_COMMAND=DEL bash /tmp/cni-failpoint-demo-helper.sh
{
"code": 999,
"msg": "oops"
}
$ sudo CNI_COMMAND=DEL bash /tmp/cni-failpoint-demo-helper.sh
{
"code": 999,
"msg": "oops"
}
$ cat /tmp/cni-failpoint.json | jq .
{
"cmdAdd": "0*error(try-again)",
"cmdDel": "0*error(oops)",
"cmdCheck": "1*off->1*panic(sorry)"
}
```
[1]: <https://www.cni.dev/plugins/current/main/bridge/>

202
integration/failpoint/cmd/cni-bridge-fp/main_linux.go Normal file
View File

@@ -0,0 +1,202 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"syscall"
"github.com/containerd/containerd/pkg/failpoint"
"github.com/containerd/continuity"
"github.com/containernetworking/cni/pkg/invoke"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/version"
)
const delegatedPlugin = "bridge"
type netConf struct {
RuntimeConfig struct {
PodAnnotations inheritedPodAnnotations `json:"io.kubernetes.cri.pod-annotations"`
} `json:"runtimeConfig,omitempty"`
}
type inheritedPodAnnotations struct {
// FailpointConfPath represents filepath of failpoint settings.
FailpointConfPath string `json:"failpoint.cni.containerd.io/confpath,omitempty"`
}
// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
type failpointConf struct {
Add string `json:"cmdAdd,omitempty"`
Del string `json:"cmdDel,omitempty"`
Check string `json:"cmdCheck,omitempty"`
}
func main() {
skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, "bridge with failpoint support")
}
func cmdAdd(args *skel.CmdArgs) error {
if err := handleFailpoint(args, "ADD"); err != nil {
return err
}
result, err := invoke.DelegateAdd(context.TODO(), delegatedPlugin, args.StdinData, nil)
if err != nil {
return err
}
return result.Print()
}
func cmdCheck(args *skel.CmdArgs) error {
if err := handleFailpoint(args, "CHECK"); err != nil {
return err
}
return invoke.DelegateCheck(context.TODO(), delegatedPlugin, args.StdinData, nil)
}
func cmdDel(args *skel.CmdArgs) error {
if err := handleFailpoint(args, "DEL"); err != nil {
return err
}
return invoke.DelegateDel(context.TODO(), delegatedPlugin, args.StdinData, nil)
}
func handleFailpoint(args *skel.CmdArgs, cmdKind string) error {
var conf netConf
if err := json.Unmarshal(args.StdinData, &conf); err != nil {
return fmt.Errorf("failed to parse network configuration: %w", err)
}
confPath := conf.RuntimeConfig.PodAnnotations.FailpointConfPath
if len(confPath) == 0 {
return nil
}
control, err := newFailpointControl(confPath)
if err != nil {
return err
}
evalFn, err := control.delegatedEvalFn(cmdKind)
if err != nil {
return err
}
return evalFn()
}
type failpointControl struct {
confPath string
}
func newFailpointControl(confPath string) (*failpointControl, error) {
if !filepath.IsAbs(confPath) {
return nil, fmt.Errorf("failpoint confPath(%s) is required to be absolute", confPath)
}
return &failpointControl{
confPath: confPath,
}, nil
}
func (c *failpointControl) delegatedEvalFn(cmdKind string) (failpoint.EvalFn, error) {
var resFn failpoint.EvalFn = nopEvalFn
if err := c.updateTx(func(conf *failpointConf) error {
var fpStr *string
switch cmdKind {
case "ADD":
fpStr = &conf.Add
case "DEL":
fpStr = &conf.Del
case "CHECK":
fpStr = &conf.Check
}
if fpStr == nil || *fpStr == "" {
return nil
}
fp, err := failpoint.NewFailpoint(cmdKind, *fpStr)
if err != nil {
return fmt.Errorf("failed to parse failpoint %s: %w", *fpStr, err)
}
resFn = fp.DelegatedEval()
*fpStr = fp.Marshal()
return nil
}); err != nil {
return nil, err
}
return resFn, nil
}
func (c *failpointControl) updateTx(updateFn func(conf *failpointConf) error) error {
f, err := os.OpenFile(c.confPath, os.O_RDWR, 0666)
if err != nil {
return fmt.Errorf("failed to open confPath %s: %w", c.confPath, err)
}
defer f.Close()
if err := flock(f.Fd()); err != nil {
return fmt.Errorf("failed to lock failpoint setting %s: %w", c.confPath, err)
}
defer unflock(f.Fd())
data, err := ioutil.ReadAll(f)
if err != nil {
return fmt.Errorf("failed to read failpoint setting %s: %w", c.confPath, err)
}
var conf failpointConf
if err := json.Unmarshal(data, &conf); err != nil {
return fmt.Errorf("failed to unmarshal failpoint conf %s: %w", string(data), err)
}
if err := updateFn(&conf); err != nil {
return err
}
data, err = json.Marshal(conf)
if err != nil {
return fmt.Errorf("failed to marshal failpoint conf: %w", err)
}
return continuity.AtomicWriteFile(c.confPath, data, 0666)
}
func nopEvalFn() error {
return nil
}
func flock(fd uintptr) error {
return syscall.Flock(int(fd), syscall.LOCK_EX)
}
func unflock(fd uintptr) error {
return syscall.Flock(int(fd), syscall.LOCK_UN)
}

29
integration/failpoint/cmd/containerd-shim-runc-fp-v1/main_linux.go Normal file
View File

@@ -0,0 +1,29 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"context"
"github.com/containerd/containerd/runtime/v2/runc/manager"
_ "github.com/containerd/containerd/runtime/v2/runc/pause"
"github.com/containerd/containerd/runtime/v2/shim"
)
func main() {
shim.RunManager(context.Background(), manager.NewShimManager("io.containerd.runc-fp.v1"))
}

141
integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin_linux.go Normal file
View File

@@ -0,0 +1,141 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
taskapi "github.com/containerd/containerd/api/runtime/task/v2"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/pkg/failpoint"
"github.com/containerd/containerd/pkg/shutdown"
"github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/runtime/v2/runc/task"
"github.com/containerd/containerd/runtime/v2/shim"
"github.com/containerd/ttrpc"
)
const (
ociConfigFilename = "config.json"
failpointPrefixKey = "io.containerd.runtime.v2.shim.failpoint."
)
func init() {
plugin.Register(&plugin.Registration{
Type: plugin.TTRPCPlugin,
ID: "task",
Requires: []plugin.Type{
plugin.EventPlugin,
plugin.InternalPlugin,
},
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
pp, err := ic.GetByID(plugin.EventPlugin, "publisher")
if err != nil {
return nil, err
}
ss, err := ic.GetByID(plugin.InternalPlugin, "shutdown")
if err != nil {
return nil, err
}
fps, err := newFailpointFromOCIAnnotation()
if err != nil {
return nil, err
}
service, err := task.NewTaskService(ic.Context, pp.(shim.Publisher), ss.(shutdown.Service))
if err != nil {
return nil, err
}
return &taskServiceWithFp{
fps: fps,
local: service,
}, nil
},
})
}
type taskServiceWithFp struct {
fps map[string]*failpoint.Failpoint
local taskapi.TaskService
}
func (s *taskServiceWithFp) RegisterTTRPC(server *ttrpc.Server) error {
taskapi.RegisterTaskService(server, s.local)
return nil
}
func (s *taskServiceWithFp) UnaryInterceptor() ttrpc.UnaryServerInterceptor {
return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) {
methodName := filepath.Base(info.FullMethod)
if fp, ok := s.fps[methodName]; ok {
if err := fp.Evaluate(); err != nil {
return nil, err
}
}
return method(ctx, unmarshal)
}
}
// newFailpointFromOCIAnnotation reloads and parses the annotation from
// bundle-path/config.json.
//
// The annotation controlling task API's failpoint should be like:
//
// io.containerd.runtime.v2.shim.failpoint.Create = 1*off->1*error(please retry)
//
// The `Create` is the shim unary API and the value of annotation is the
// failpoint control. The function will return a set of failpoint controllers.
func newFailpointFromOCIAnnotation() (map[string]*failpoint.Failpoint, error) {
// NOTE: shim's current working dir is in bundle dir.
cwd, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("failed to get current working dir: %w", err)
}
configPath := filepath.Join(cwd, ociConfigFilename)
data, err := os.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to read %v: %w", configPath, err)
}
var spec oci.Spec
if err := json.Unmarshal(data, &spec); err != nil {
return nil, fmt.Errorf("failed to parse oci.Spec(%v): %w", string(data), err)
}
res := make(map[string]*failpoint.Failpoint)
for k, v := range spec.Annotations {
if !strings.HasPrefix(k, failpointPrefixKey) {
continue
}
methodName := strings.TrimPrefix(k, failpointPrefixKey)
fp, err := failpoint.NewFailpoint(methodName, v)
if err != nil {
return nil, fmt.Errorf("failed to parse failpoint %v: %w", v, err)
}
res[methodName] = fp
}
return res, nil
}

3
integration/main_test.go
View File

@@ -174,7 +174,8 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb
Uid: util.GenerateID(),
Namespace: Randomize(ns),
},
Linux: &runtime.LinuxPodSandboxConfig{},
Linux: &runtime.LinuxPodSandboxConfig{},
Annotations: make(map[string]string),
}
for _, opt := range opts {
opt(config)

122
integration/sandbox_run_rollback_test.go Normal file
View File

@@ -0,0 +1,122 @@
//go:build linux
// +build linux
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/failpoint"
"github.com/stretchr/testify/require"
)
const (
failpointRuntimeHandler = "runc-fp"
failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint."
failpointCNIConfPathKey = "failpoint.cni.containerd.io/confpath"
)
func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip()
}
t.Logf("Inject CNI failpoint")
conf := &failpointConf{
Add: "1*error(you-shall-not-pass!)",
}
sbConfig := PodSandboxConfig(t.Name(), "failpoint")
injectCNIFailpoint(t, sbConfig, conf)
t.Logf("Create a sandbox")
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
require.Error(t, err)
require.Equal(t, true, strings.Contains(err.Error(), "you-shall-not-pass!"))
t.Logf("Retry to create sandbox with same config")
sb, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
require.NoError(t, err)
err = runtimeService.StopPodSandbox(sb)
require.NoError(t, err)
err = runtimeService.RemovePodSandbox(sb)
require.NoError(t, err)
}
func TestRunPodSandboxWithShimStartFailure(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip()
}
t.Logf("Inject Shim failpoint")
sbConfig := PodSandboxConfig(t.Name(), "failpoint")
injectShimFailpoint(t, sbConfig, map[string]string{
"Start": "1*error(no hard feelings)",
})
t.Logf("Create a sandbox")
_, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler)
require.Error(t, err)
require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings"))
}
// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint.
type failpointConf struct {
Add string `json:"cmdAdd"`
Del string `json:"cmdDel"`
Check string `json:"cmdCheck"`
}
func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf *failpointConf) {
stateDir := t.TempDir()
metadata := sbConfig.Metadata
fpFilename := filepath.Join(stateDir,
fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name))
data, err := json.Marshal(conf)
require.NoError(t, err)
err = os.WriteFile(fpFilename, data, 0666)
require.NoError(t, err)
sbConfig.Annotations[failpointCNIConfPathKey] = fpFilename
}
func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) {
for method, fp := range methodFps {
_, err := failpoint.NewFailpoint(method, fp)
require.NoError(t, err, "check failpoint %s for shim method %s", fp, method)
sbConfig.Annotations[failpointShimPrefixKey+method] = fp
}
}

310
pkg/failpoint/fail.go Normal file
View File

@@ -0,0 +1,310 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package failpoint provides the code point in the path, which can be controlled
// by user's variable.
//
// Inspired by FreeBSD fail(9): https://freebsd.org/cgi/man.cgi?query=fail.
package failpoint
import (
"bytes"
"fmt"
"strconv"
"strings"
"sync"
"time"
)
// EvalFn is the func type about delegated evaluation.
type EvalFn func() error
// Type is the type of failpoint to specifies which action to take.
type Type int
const (
// TypeInvalid is invalid type
TypeInvalid Type = iota
// TypeOff takes no action
TypeOff
// TypeError triggers failpoint error with specified argument
TypeError
// TypePanic triggers panic with specified argument
TypePanic
// TypeDelay sleeps with the specified number of milliseconds
TypeDelay
)
// String returns the name of type.
func (t Type) String() string {
switch t {
case TypeOff:
return "off"
case TypeError:
return "error"
case TypePanic:
return "panic"
case TypeDelay:
return "delay"
default:
return "invalid"
}
}
// Failpoint is used to add code points where error or panic may be injected by
// user. The user controlled variable will be parsed for how the error injected
// code should fire. There is the way to set the rule for failpoint.
//
// <count>*<type>[(arg)][-><more terms>]
//
// The <type> argument specifies which action to take; it can be one of:
//
// off: Takes no action (does not trigger failpoint and no argument)
// error: Triggers failpoint error with specified argument(string)
// panic: Triggers panic with specified argument(string)
// delay: Sleep the specified number of milliseconds
//
// The <count>* modifiers prior to <type> control when <type> is executed. For
// example, "5*error(oops)" means "return error oops 5 times total". The
// operator -> can be used to express cascading terms. If you specify
// <term1>-><term2>, it means that if <term1> does not execute, <term2> will
// be evaluated. If you want the error injected code should fire in second
// call, you can specify "1*off->1*error(oops)".
//
// Inspired by FreeBSD fail(9): https://freebsd.org/cgi/man.cgi?query=fail.
type Failpoint struct {
sync.Mutex
fnName string
entries []*failpointEntry
}
// NewFailpoint returns failpoint control.
func NewFailpoint(fnName string, terms string) (*Failpoint, error) {
entries, err := parseTerms([]byte(terms))
if err != nil {
return nil, err
}
return &Failpoint{
fnName: fnName,
entries: entries,
}, nil
}
// Evaluate evaluates a failpoint.
func (fp *Failpoint) Evaluate() error {
fn := fp.DelegatedEval()
return fn()
}
// DelegatedEval evaluates a failpoint but delegates to caller to fire that.
func (fp *Failpoint) DelegatedEval() EvalFn {
var target *failpointEntry
func() {
fp.Lock()
defer fp.Unlock()
for _, entry := range fp.entries {
if entry.count == 0 {
continue
}
entry.count--
target = entry
break
}
}()
if target == nil {
return nopEvalFn
}
return target.evaluate
}
// Failpoint returns the current state of control in string format.
func (fp *Failpoint) Marshal() string {
fp.Lock()
defer fp.Unlock()
res := make([]string, 0, len(fp.entries))
for _, entry := range fp.entries {
res = append(res, entry.marshal())
}
return strings.Join(res, "->")
}
type failpointEntry struct {
typ Type
arg interface{}
count int64
}
func newFailpointEntry() *failpointEntry {
return &failpointEntry{
typ: TypeInvalid,
count: 0,
}
}
func (fpe *failpointEntry) marshal() string {
base := fmt.Sprintf("%d*%s", fpe.count, fpe.typ)
switch fpe.typ {
case TypeOff:
return base
case TypeError, TypePanic:
return fmt.Sprintf("%s(%s)", base, fpe.arg.(string))
case TypeDelay:
return fmt.Sprintf("%s(%d)", base, fpe.arg.(time.Duration)/time.Millisecond)
default:
return base
}
}
func (fpe *failpointEntry) evaluate() error {
switch fpe.typ {
case TypeOff:
return nil
case TypeError:
return fmt.Errorf("%v", fpe.arg)
case TypePanic:
panic(fpe.arg)
case TypeDelay:
time.Sleep(fpe.arg.(time.Duration))
return nil
default:
panic("invalid failpoint type")
}
}
func parseTerms(term []byte) ([]*failpointEntry, error) {
var entry *failpointEntry
var err error
// count*type[(arg)]
term, entry, err = parseTerm(term)
if err != nil {
return nil, err
}
res := []*failpointEntry{entry}
// cascading terms
for len(term) > 0 {
if !bytes.HasPrefix(term, []byte("->")) {
return nil, fmt.Errorf("invalid cascading terms: %s", string(term))
}
term = term[2:]
term, entry, err = parseTerm(term)
if err != nil {
return nil, fmt.Errorf("failed to parse cascading term: %w", err)
}
res = append(res, entry)
}
return res, nil
}
func parseTerm(term []byte) ([]byte, *failpointEntry, error) {
var err error
var entry = newFailpointEntry()
// count*
term, err = parseInt64(term, '*', &entry.count)
if err != nil {
return nil, nil, err
}
// type[(arg)]
term, err = parseType(term, entry)
return term, entry, err
}
func parseType(term []byte, entry *failpointEntry) ([]byte, error) {
var nameToTyp = map[string]Type{
"off": TypeOff,
"error(": TypeError,
"panic(": TypePanic,
"delay(": TypeDelay,
}
var found bool
for name, typ := range nameToTyp {
if bytes.HasPrefix(term, []byte(name)) {
found = true
term = term[len(name):]
entry.typ = typ
break
}
}
if !found {
return nil, fmt.Errorf("invalid type format: %s", string(term))
}
switch entry.typ {
case TypePanic, TypeError:
endIdx := bytes.IndexByte(term, ')')
if endIdx <= 0 {
return nil, fmt.Errorf("invalid argument for %s type", entry.typ)
}
entry.arg = string(term[:endIdx])
return term[endIdx+1:], nil
case TypeOff:
// do nothing
return term, nil
case TypeDelay:
var msVal int64
var err error
term, err = parseInt64(term, ')', &msVal)
if err != nil {
return nil, err
}
entry.arg = time.Millisecond * time.Duration(msVal)
return term, nil
default:
panic("unreachable")
}
}
func parseInt64(term []byte, terminate byte, val *int64) ([]byte, error) {
i := 0
for ; i < len(term); i++ {
if b := term[i]; b < '0' || b > '9' {
break
}
}
if i == 0 || i == len(term) || term[i] != terminate {
return nil, fmt.Errorf("failed to parse int64 because of invalid terminate byte: %s", string(term))
}
v, err := strconv.ParseInt(string(term[:i]), 10, 64)
if err != nil {
return nil, fmt.Errorf("failed to parse int64 from %s: %v", string(term[:i]), err)
}
*val = v
return term[i+1:], nil
}
func nopEvalFn() error {
return nil
}

134
pkg/failpoint/fail_test.go Normal file
View File

@@ -0,0 +1,134 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package failpoint
import (
"reflect"
"testing"
"time"
)
func TestParseTerms(t *testing.T) {
cases := []struct {
terms string
hasError bool
}{
// off
{"5", true},
{"*off()", true},
{"5*off()", true},
{"5*off(nothing)", true},
{"5*off(", true},
{"5*off", false},
// error
{"10000error(oops)", true},
{"10*error(oops)", false},
{"1234*error(oops))", true},
{"12342*error()", true},
// panic
{"1panic(oops)", true},
{"1000000*panic(oops)", false},
{"12345*panic(oops))", true},
{"12*panic()", true},
// delay
{"1*delay(oops)", true},
{"1000000*delay(-1)", true},
{"1000000*delay(1)", false},
// cascading terms
{"1*delay(1)-", true},
{"10*delay(2)->", true},
{"11*delay(3)->10*off(", true},
{"12*delay(4)->10*of", true},
{"13*delay(5)->10*off->1000*panic(oops)", false},
}
for i, c := range cases {
fp, err := NewFailpoint(t.Name(), c.terms)
if (err != nil && !c.hasError) ||
(err == nil && c.hasError) {
t.Fatalf("[%v - %s] expected hasError=%v, but got %v", i, c.terms, c.hasError, err)
}
if err != nil {
continue
}
if got := fp.Marshal(); !reflect.DeepEqual(got, c.terms) {
t.Fatalf("[%v] expected %v, but got %v", i, c.terms, got)
}
}
}
func TestEvaluate(t *testing.T) {
terms := "1*error(oops-)->1*off->1*delay(1000)->1*panic(panic)"
fp, err := NewFailpoint(t.Name(), terms)
if err != nil {
t.Fatalf("unexpected error %v", err)
}
injectedFn := func() error {
if err := fp.Evaluate(); err != nil {
return err
}
return nil
}
// should return oops- error
if err := injectedFn(); err == nil || err.Error() != "oops-" {
t.Fatalf("expected error %v, but got %v", "oops-", err)
}
// should return nil
if err := injectedFn(); err != nil {
t.Fatalf("expected nil, but got %v", err)
}
// should sleep 1s and return nil
now := time.Now()
err = injectedFn()
du := time.Since(now)
if err != nil {
t.Fatalf("expected nil, but got %v", err)
}
if du < 1*time.Second {
t.Fatalf("expected sleep 1s, but got %v", du)
}
// should panic
defer func() {
if err := recover(); err == nil || err.(string) != "panic" {
t.Fatalf("should panic(panic), but got %v", err)
}
expected := "0*error(oops-)->0*off->0*delay(1000)->0*panic(panic)"
if got := fp.Marshal(); got != expected {
t.Fatalf("expected %v, but got %v", expected, got)
}
if err := injectedFn(); err != nil {
t.Fatalf("expected nil, but got %v", err)
}
}()
injectedFn()
}

25
runtime/v2/shim/shim.go
View File

@@ -108,6 +108,12 @@ type ttrpcService interface {
RegisterTTRPC(*ttrpc.Server) error
}
type ttrpcServerOptioner interface {
ttrpcService
UnaryInterceptor() ttrpc.UnaryServerInterceptor
}
type taskService struct {
shimapi.TaskService
}
@@ -370,6 +376,8 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi
var (
initialized = plugin.NewPluginSet()
ttrpcServices = []ttrpcService{}
ttrpcUnaryInterceptors = []ttrpc.UnaryServerInterceptor{}
)
plugins := plugin.Graph(func(*plugin.Registration) bool { return false })
for _, p := range plugins {
@@ -409,19 +417,28 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi
if err != nil {
if plugin.IsSkipPlugin(err) {
log.G(ctx).WithError(err).WithField("type", p.Type).Infof("skip loading plugin %q...", id)
} else {
log.G(ctx).WithError(err).Warnf("failed to load plugin %s", id)
continue
}
continue
return fmt.Errorf("failed to load plugin %s: %w", id, err)
}
if src, ok := instance.(ttrpcService); ok {
logrus.WithField("id", id).Debug("registering ttrpc service")
ttrpcServices = append(ttrpcServices, src)
}
if src, ok := instance.(ttrpcServerOptioner); ok {
ttrpcUnaryInterceptors = append(ttrpcUnaryInterceptors, src.UnaryInterceptor())
}
}
server, err := newServer()
if len(ttrpcServices) == 0 {
return fmt.Errorf("required that ttrpc service")
}
unaryInterceptor := chainUnaryServerInterceptors(ttrpcUnaryInterceptors...)
server, err := newServer(ttrpc.WithUnaryServerInterceptor(unaryInterceptor))
if err != nil {
return fmt.Errorf("failed creating server: %w", err)
}

4
runtime/v2/shim/shim_darwin.go
View File

@@ -18,8 +18,8 @@ package shim
import "github.com/containerd/ttrpc"
func newServer() (*ttrpc.Server, error) {
return ttrpc.NewServer()
func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) {
return ttrpc.NewServer(opts...)
}
func subreaper() error {

4
runtime/v2/shim/shim_freebsd.go
View File

@@ -18,8 +18,8 @@ package shim
import "github.com/containerd/ttrpc"
func newServer() (*ttrpc.Server, error) {
return ttrpc.NewServer()
func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) {
return ttrpc.NewServer(opts...)
}
func subreaper() error {

5
runtime/v2/shim/shim_linux.go
View File

@@ -21,8 +21,9 @@ import (
"github.com/containerd/ttrpc"
)
func newServer() (*ttrpc.Server, error) {
return ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser()))
func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) {
opts = append(opts, ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser()))
return ttrpc.NewServer(opts...)
}
func subreaper() error {

2
runtime/v2/shim/shim_windows.go
View File

@@ -31,7 +31,7 @@ func setupSignals(config Config) (chan os.Signal, error) {
return nil, errors.New("not supported")
}
func newServer() (*ttrpc.Server, error) {
func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) {
return nil, errors.New("not supported")
}

26
runtime/v2/shim/util.go
View File

@@ -30,6 +30,7 @@ import (
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/protobuf/proto"
"github.com/containerd/containerd/protobuf/types"
"github.com/containerd/ttrpc"
exec "golang.org/x/sys/execabs"
)
@@ -167,3 +168,28 @@ func ReadAddress(path string) (string, error) {
}
return string(data), nil
}
// chainUnaryServerInterceptors creates a single ttrpc server interceptor from
// a chain of many interceptors executed from first to last.
func chainUnaryServerInterceptors(interceptors ...ttrpc.UnaryServerInterceptor) ttrpc.UnaryServerInterceptor {
n := len(interceptors)
// force to use default interceptor in ttrpc
if n == 0 {
return nil
}
return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) {
currentMethod := method
for i := n - 1; i > 0; i-- {
interceptor := interceptors[i]
innerMethod := currentMethod
currentMethod = func(currentCtx context.Context, currentUnmarshal func(interface{}) error) (interface{}, error) {
return interceptor(currentCtx, currentUnmarshal, info, innerMethod)
}
}
return interceptors[0](ctx, unmarshal, info, currentMethod)
}
}

118
runtime/v2/shim/util_test.go Normal file
View File

@@ -0,0 +1,118 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package shim
import (
"context"
"path/filepath"
"reflect"
"testing"
"github.com/containerd/ttrpc"
)
func TestChainUnaryServerInterceptors(t *testing.T) {
methodInfo := &ttrpc.UnaryServerInfo{
FullMethod: filepath.Join("/", t.Name(), "foo"),
}
type callKey struct{}
callValue := "init"
callCtx := context.WithValue(context.Background(), callKey{}, callValue)
verifyCallCtxFn := func(ctx context.Context, key interface{}, expected interface{}) {
got := ctx.Value(key)
if !reflect.DeepEqual(expected, got) {
t.Fatalf("[context(key:%s) expected %v, but got %v", key, expected, got)
}
}
verifyInfoFn := func(info *ttrpc.UnaryServerInfo) {
if !reflect.DeepEqual(methodInfo, info) {
t.Fatalf("[info] expected %+v, but got %+v", methodInfo, info)
}
}
origUnmarshaler := func(obj interface{}) error {
v := obj.(*int64)
*v *= 2
return nil
}
type firstKey struct{}
firstValue := "from first"
var firstUnmarshaler ttrpc.Unmarshaler
first := func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) {
verifyCallCtxFn(ctx, callKey{}, callValue)
verifyInfoFn(info)
ctx = context.WithValue(ctx, firstKey{}, firstValue)
firstUnmarshaler = func(obj interface{}) error {
if err := unmarshal(obj); err != nil {
return err
}
v := obj.(*int64)
*v *= 2
return nil
}
return method(ctx, firstUnmarshaler)
}
type secondKey struct{}
secondValue := "from second"
second := func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) {
verifyCallCtxFn(ctx, callKey{}, callValue)
verifyCallCtxFn(ctx, firstKey{}, firstValue)
verifyInfoFn(info)
v := int64(3) // should return 12
if err := unmarshal(&v); err != nil {
t.Fatalf("unexpected error %v", err)
}
if expected := int64(12); v != expected {
t.Fatalf("expected int64(%v), but got %v", expected, v)
}
ctx = context.WithValue(ctx, secondKey{}, secondValue)
return method(ctx, unmarshal)
}
methodFn := func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
verifyCallCtxFn(ctx, callKey{}, callValue)
verifyCallCtxFn(ctx, firstKey{}, firstValue)
verifyCallCtxFn(ctx, secondKey{}, secondValue)
v := int64(2)
if err := unmarshal(&v); err != nil {
return nil, err
}
return v, nil
}
interceptor := chainUnaryServerInterceptors(first, second)
v, err := interceptor(callCtx, origUnmarshaler, methodInfo, methodFn)
if err != nil {
t.Fatalf("expected nil, but got %v", err)
}
if expected := int64(8); v != expected {
t.Fatalf("expected result is int64(%v), but got %v", expected, v)
}
}

35
script/setup/install-failpoint-binaries Executable file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Copyright The containerd Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Build and install
#
# * cni-bridge-fp into /opt/cni/bin
# * containerd-shim-runc-fp-v1 into /usr/local/bin
#
set -euo pipefail
base_dir="$(dirname "${BASH_SOURCE[0]}")"
root_dir="$( cd "${base_dir}" && pwd )"/../..
cd "${root_dir}"
CNI_BIN_DIR=${CNI_BIN_DIR:-"/opt/cni/bin"}
make bin/cni-bridge-fp
sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}"
SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"}
make bin/containerd-shim-runc-fp-v1
sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}"

58
script/test/utils.sh
View File

@@ -35,9 +35,10 @@ CONTAINERD_RUNTIME=${CONTAINERD_RUNTIME:-""}
if [ -z "${CONTAINERD_CONFIG_FILE}" ]; then
config_file="${CONTAINERD_CONFIG_DIR}/containerd-config-cri.toml"
truncate --size 0 "${config_file}"
echo "version=2" >> ${config_file}
if command -v sestatus >/dev/null 2>&1; then
cat >>${config_file} <<EOF
version=2
[plugins."io.containerd.grpc.v1.cri"]
enable_selinux = true
EOF
@@ -51,6 +52,59 @@ EOF
CONTAINERD_CONFIG_FILE="${config_file}"
fi
if [ $IS_WINDOWS -eq 0 ]; then
FAILPOINT_CONTAINERD_RUNTIME="runc-fp.v1"
FAILPOINT_CNI_CONF_DIR=${FAILPOINT_CNI_CONF_DIR:-"/tmp/failpoint-cni-net.d"}
mkdir -p "${FAILPOINT_CNI_CONF_DIR}"
# Add runtime with failpoint
cat << EOF | tee -a "${CONTAINERD_CONFIG_FILE}"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc-fp]
cni_conf_dir = "${FAILPOINT_CNI_CONF_DIR}"
cni_max_conf_num = 1
pod_annotations = ["io.containerd.runtime.v2.shim.failpoint.*"]
runtime_type = "${FAILPOINT_CONTAINERD_RUNTIME}"
EOF
cat << EOF | tee "${FAILPOINT_CNI_CONF_DIR}/10-containerd-net.conflist"
{
"cniVersion": "1.0.0",
"name": "containerd-net-failpoint",
"plugins": [
{
"type": "cni-bridge-fp",
"bridge": "cni-fp",
"isGateway": true,
"ipMasq": true,
"promiscMode": true,
"ipam": {
"type": "host-local",
"ranges": [
[{
"subnet": "10.88.0.0/16"
}],
[{
"subnet": "2001:4860:4860::/64"
}]
],
"routes": [
{ "dst": "0.0.0.0/0" },
{ "dst": "::/0" }
]
},
"capabilities": {
"io.kubernetes.cri.pod-annotations": true
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true}
}
]
}
EOF
fi
# CONTAINERD_TEST_SUFFIX is the suffix appended to the root/state directory used
# by test containerd.
CONTAINERD_TEST_SUFFIX=${CONTAINERD_TEST_SUFFIX:-"-test"}
@@ -183,6 +237,8 @@ test_setup() {
fi
readiness_check run_ctr
readiness_check run_crictl
# Show the config about cri plugin in log when it's ready
run_crictl
}
# test_teardown kills containerd.

294
vendor/github.com/containernetworking/cni/pkg/skel/skel.go generated vendored Normal file
View File

@@ -0,0 +1,294 @@
// Copyright 2014-2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package skel provides skeleton code for a CNI plugin.
// In particular, it implements argument parsing and validation.
package skel
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strings"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/utils"
"github.com/containernetworking/cni/pkg/version"
)
// CmdArgs captures all the arguments passed in to the plugin
// via both env vars and stdin
type CmdArgs struct {
ContainerID string
Netns string
IfName string
Args string
Path string
StdinData []byte
}
type dispatcher struct {
Getenv func(string) string
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
ConfVersionDecoder version.ConfigDecoder
VersionReconciler version.Reconciler
}
type reqForCmdEntry map[string]bool
func (t *dispatcher) getCmdArgsFromEnv() (string, *CmdArgs, *types.Error) {
var cmd, contID, netns, ifName, args, path string
vars := []struct {
name string
val *string
reqForCmd reqForCmdEntry
}{
{
"CNI_COMMAND",
&cmd,
reqForCmdEntry{
"ADD": true,
"CHECK": true,
"DEL": true,
},
},
{
"CNI_CONTAINERID",
&contID,
reqForCmdEntry{
"ADD": true,
"CHECK": true,
"DEL": true,
},
},
{
"CNI_NETNS",
&netns,
reqForCmdEntry{
"ADD": true,
"CHECK": true,
"DEL": false,
},
},
{
"CNI_IFNAME",
&ifName,
reqForCmdEntry{
"ADD": true,
"CHECK": true,
"DEL": true,
},
},
{
"CNI_ARGS",
&args,
reqForCmdEntry{
"ADD": false,
"CHECK": false,
"DEL": false,
},
},
{
"CNI_PATH",
&path,
reqForCmdEntry{
"ADD": true,
"CHECK": true,
"DEL": true,
},
},
}
argsMissing := make([]string, 0)
for _, v := range vars {
*v.val = t.Getenv(v.name)
if *v.val == "" {
if v.reqForCmd[cmd] || v.name == "CNI_COMMAND" {
argsMissing = append(argsMissing, v.name)
}
}
}
if len(argsMissing) > 0 {
joined := strings.Join(argsMissing, ",")
return "", nil, types.NewError(types.ErrInvalidEnvironmentVariables, fmt.Sprintf("required env variables [%s] missing", joined), "")
}
if cmd == "VERSION" {
t.Stdin = bytes.NewReader(nil)
}
stdinData, err := ioutil.ReadAll(t.Stdin)
if err != nil {
return "", nil, types.NewError(types.ErrIOFailure, fmt.Sprintf("error reading from stdin: %v", err), "")
}
cmdArgs := &CmdArgs{
ContainerID: contID,
Netns: netns,
IfName: ifName,
Args: args,
Path: path,
StdinData: stdinData,
}
return cmd, cmdArgs, nil
}
func (t *dispatcher) checkVersionAndCall(cmdArgs *CmdArgs, pluginVersionInfo version.PluginInfo, toCall func(*CmdArgs) error) *types.Error {
configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData)
if err != nil {
return types.NewError(types.ErrDecodingFailure, err.Error(), "")
}
verErr := t.VersionReconciler.Check(configVersion, pluginVersionInfo)
if verErr != nil {
return types.NewError(types.ErrIncompatibleCNIVersion, "incompatible CNI versions", verErr.Details())
}
if err = toCall(cmdArgs); err != nil {
if e, ok := err.(*types.Error); ok {
// don't wrap Error in Error
return e
}
return types.NewError(types.ErrInternal, err.Error(), "")
}
return nil
}
func validateConfig(jsonBytes []byte) *types.Error {
var conf struct {
Name string `json:"name"`
}
if err := json.Unmarshal(jsonBytes, &conf); err != nil {
return types.NewError(types.ErrDecodingFailure, fmt.Sprintf("error unmarshall network config: %v", err), "")
}
if conf.Name == "" {
return types.NewError(types.ErrInvalidNetworkConfig, "missing network name", "")
}
if err := utils.ValidateNetworkName(conf.Name); err != nil {
return err
}
return nil
}
func (t *dispatcher) pluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) *types.Error {
cmd, cmdArgs, err := t.getCmdArgsFromEnv()
if err != nil {
// Print the about string to stderr when no command is set
if err.Code == types.ErrInvalidEnvironmentVariables && t.Getenv("CNI_COMMAND") == "" && about != "" {
_, _ = fmt.Fprintln(t.Stderr, about)
_, _ = fmt.Fprintf(t.Stderr, "CNI protocol versions supported: %s\n", strings.Join(versionInfo.SupportedVersions(), ", "))
return nil
}
return err
}
if cmd != "VERSION" {
if err = validateConfig(cmdArgs.StdinData); err != nil {
return err
}
if err = utils.ValidateContainerID(cmdArgs.ContainerID); err != nil {
return err
}
if err = utils.ValidateInterfaceName(cmdArgs.IfName); err != nil {
return err
}
}
switch cmd {
case "ADD":
err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdAdd)
case "CHECK":
configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData)
if err != nil {
return types.NewError(types.ErrDecodingFailure, err.Error(), "")
}
if gtet, err := version.GreaterThanOrEqualTo(configVersion, "0.4.0"); err != nil {
return types.NewError(types.ErrDecodingFailure, err.Error(), "")
} else if !gtet {
return types.NewError(types.ErrIncompatibleCNIVersion, "config version does not allow CHECK", "")
}
for _, pluginVersion := range versionInfo.SupportedVersions() {
gtet, err := version.GreaterThanOrEqualTo(pluginVersion, configVersion)
if err != nil {
return types.NewError(types.ErrDecodingFailure, err.Error(), "")
} else if gtet {
if err := t.checkVersionAndCall(cmdArgs, versionInfo, cmdCheck); err != nil {
return err
}
return nil
}
}
return types.NewError(types.ErrIncompatibleCNIVersion, "plugin version does not allow CHECK", "")
case "DEL":
err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdDel)
case "VERSION":
if err := versionInfo.Encode(t.Stdout); err != nil {
return types.NewError(types.ErrIOFailure, err.Error(), "")
}
default:
return types.NewError(types.ErrInvalidEnvironmentVariables, fmt.Sprintf("unknown CNI_COMMAND: %v", cmd), "")
}
return err
}
// PluginMainWithError is the core "main" for a plugin. It accepts
// callback functions for add, check, and del CNI commands and returns an error.
//
// The caller must also specify what CNI spec versions the plugin supports.
//
// It is the responsibility of the caller to check for non-nil error return.
//
// For a plugin to comply with the CNI spec, it must print any error to stdout
// as JSON and then exit with nonzero status code.
//
// To let this package automatically handle errors and call os.Exit(1) for you,
// use PluginMain() instead.
func PluginMainWithError(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) *types.Error {
return (&dispatcher{
Getenv: os.Getenv,
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}).pluginMain(cmdAdd, cmdCheck, cmdDel, versionInfo, about)
}
// PluginMain is the core "main" for a plugin which includes automatic error handling.
//
// The caller must also specify what CNI spec versions the plugin supports.
//
// The caller can specify an "about" string, which is printed on stderr
// when no CNI_COMMAND is specified. The recommended output is "CNI plugin <foo> v<version>"
//
// When an error occurs in either cmdAdd, cmdCheck, or cmdDel, PluginMain will print the error
// as JSON to stdout and call os.Exit(1).
//
// To have more control over error handling, use PluginMainWithError() instead.
func PluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) {
if e := PluginMainWithError(cmdAdd, cmdCheck, cmdDel, versionInfo, about); e != nil {
if err := e.Print(); err != nil {
log.Print("Error writing error JSON to stdout: ", err)
}
os.Exit(1)
}
}

1
vendor/modules.txt vendored
View File

@@ -134,6 +134,7 @@ github.com/containerd/zfs/plugin
## explicit; go 1.14
github.com/containernetworking/cni/libcni
github.com/containernetworking/cni/pkg/invoke
github.com/containernetworking/cni/pkg/skel
github.com/containernetworking/cni/pkg/types
github.com/containernetworking/cni/pkg/types/020
github.com/containernetworking/cni/pkg/types/040