From ffd59ba6001af605e57c08a5fc07c8337954e190 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Thu, 16 Jun 2022 23:59:30 +0800 Subject: [PATCH 1/9] pkg/failpoint: init failpoint package Failpoint is used to control the fail during API call when testing, especially the API is complicated like CRI-RunPodSandbox. It can help us to test the unexpected behavior without mock. The control design is based on freebsd fail(9), but simpler. REF: https://www.freebsd.org/cgi/man.cgi?query=fail&sektion=9&apropos=0&manpath=FreeBSD%2B10.0-RELEASE Signed-off-by: Wei Fu --- pkg/failpoint/fail.go | 293 +++++++++++++++++++++++++++++++++++++ pkg/failpoint/fail_test.go | 134 +++++++++++++++++ 2 files changed, 427 insertions(+) create mode 100644 pkg/failpoint/fail.go create mode 100644 pkg/failpoint/fail_test.go diff --git a/pkg/failpoint/fail.go b/pkg/failpoint/fail.go new file mode 100644 index 000000000..7056f5c41 --- /dev/null +++ b/pkg/failpoint/fail.go @@ -0,0 +1,293 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package failpoint + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "sync" + "time" +) + +// Type is the type of failpoint to specifies which action to take. +type Type int + +const ( + // TypeInvalid is invalid type + TypeInvalid Type = iota + // TypeOff takes no action + TypeOff + // TypeError triggers failpoint error with specified argument + TypeError + // TypePanic triggers panic with specified argument + TypePanic + // TypeDelay sleeps with the specified number of milliseconds + TypeDelay +) + +// String returns the name of type. +func (t Type) String() string { + switch t { + case TypeOff: + return "off" + case TypeError: + return "error" + case TypePanic: + return "panic" + case TypeDelay: + return "delay" + default: + return "invalid" + } +} + +// Failpoint is used to add code points where error or panic may be injected by +// user. The user controlled variable will be parsed for how the error injected +// code should fire. There is the way to set the rule for failpoint. +// +// *[(arg)][->] +// +// The argument specifies which action to take; it can be one of: +// +// off: Takes no action (does not trigger failpoint and no argument) +// error: Triggers failpoint error with specified argument(string) +// panic: Triggers panic with specified argument(string) +// delay: Sleep the specified number of milliseconds +// +// The * modifiers prior to control when is executed. For +// example, "5*error(oops)" means "return error oops 5 times total". The +// operator -> can be used to express cascading terms. If you specify +// ->, it means that if does not execute, will +// be evaluated. If you want the error injected code should fire in second +// call, you can specify "1*off->1*error(oops)". +// +// Based on fail(9) freebsd: https://www.freebsd.org/cgi/man.cgi?query=fail&sektion=9&apropos=0&manpath=FreeBSD%2B10.0-RELEASE +type Failpoint struct { + sync.Mutex + + fnName string + entries []*failpointEntry +} + +// NewFailpoint returns failpoint control. +func NewFailpoint(fnName string, terms string) (*Failpoint, error) { + entries, err := parseTerms([]byte(terms)) + if err != nil { + return nil, err + } + + return &Failpoint{ + fnName: fnName, + entries: entries, + }, nil +} + +// Evaluate evaluates a failpoint. +func (fp *Failpoint) Evaluate() error { + var target *failpointEntry + + func() { + fp.Lock() + defer fp.Unlock() + + for _, entry := range fp.entries { + if entry.count == 0 { + continue + } + + entry.count-- + target = entry + break + } + }() + + if target == nil { + return nil + } + return target.evaluate() +} + +// Failpoint returns the current state of control in string format. +func (fp *Failpoint) Marshal() string { + fp.Lock() + defer fp.Unlock() + + res := make([]string, 0, len(fp.entries)) + for _, entry := range fp.entries { + res = append(res, entry.marshal()) + } + return strings.Join(res, "->") +} + +type failpointEntry struct { + typ Type + arg interface{} + count int64 +} + +func newFailpointEntry() *failpointEntry { + return &failpointEntry{ + typ: TypeInvalid, + count: 0, + } +} + +func (fpe *failpointEntry) marshal() string { + base := fmt.Sprintf("%d*%s", fpe.count, fpe.typ) + switch fpe.typ { + case TypeOff: + return base + case TypeError, TypePanic: + return fmt.Sprintf("%s(%s)", base, fpe.arg.(string)) + case TypeDelay: + return fmt.Sprintf("%s(%d)", base, fpe.arg.(time.Duration)/time.Millisecond) + default: + return base + } +} + +func (fpe *failpointEntry) evaluate() error { + switch fpe.typ { + case TypeOff: + return nil + case TypeError: + return fmt.Errorf("%v", fpe.arg) + case TypePanic: + panic(fpe.arg) + case TypeDelay: + time.Sleep(fpe.arg.(time.Duration)) + return nil + default: + panic("invalid failpoint type") + } +} + +func parseTerms(term []byte) ([]*failpointEntry, error) { + var entry *failpointEntry + var err error + + // count*type[(arg)] + term, entry, err = parseTerm(term) + if err != nil { + return nil, err + } + + res := []*failpointEntry{entry} + + // cascading terms + for len(term) > 0 { + if !bytes.HasPrefix(term, []byte("->")) { + return nil, fmt.Errorf("invalid cascading terms: %s", string(term)) + } + + term = term[2:] + term, entry, err = parseTerm(term) + if err != nil { + return nil, fmt.Errorf("failed to parse cascading term: %w", err) + } + + res = append(res, entry) + } + return res, nil +} + +func parseTerm(term []byte) ([]byte, *failpointEntry, error) { + var err error + var entry = newFailpointEntry() + + // count* + term, err = parseInt64(term, '*', &entry.count) + if err != nil { + return nil, nil, err + } + + // type[(arg)] + term, err = parseType(term, entry) + return term, entry, err +} + +func parseType(term []byte, entry *failpointEntry) ([]byte, error) { + var nameToTyp = map[string]Type{ + "off": TypeOff, + "error(": TypeError, + "panic(": TypePanic, + "delay(": TypeDelay, + } + + var found bool + for name, typ := range nameToTyp { + if bytes.HasPrefix(term, []byte(name)) { + found = true + term = term[len(name):] + entry.typ = typ + break + } + } + + if !found { + return nil, fmt.Errorf("invalid type format: %s", string(term)) + } + + switch entry.typ { + case TypePanic, TypeError: + endIdx := bytes.IndexByte(term, ')') + if endIdx <= 0 { + return nil, fmt.Errorf("invalid argument for %s type", entry.typ) + } + entry.arg = string(term[:endIdx]) + return term[endIdx+1:], nil + case TypeOff: + // do nothing + return term, nil + case TypeDelay: + var msVal int64 + var err error + + term, err = parseInt64(term, ')', &msVal) + if err != nil { + return nil, err + } + entry.arg = time.Millisecond * time.Duration(msVal) + return term, nil + default: + panic("unreachable") + } +} + +func parseInt64(term []byte, terminate byte, val *int64) ([]byte, error) { + i := 0 + + for ; i < len(term); i++ { + if b := term[i]; b < '0' || b > '9' { + break + } + } + + if i == 0 || i == len(term) || term[i] != terminate { + return nil, fmt.Errorf("failed to parse int64 because of invalid terminate byte: %s", string(term)) + } + + v, err := strconv.ParseInt(string(term[:i]), 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse int64 from %s: %v", string(term[:i]), err) + } + + *val = v + return term[i+1:], nil +} diff --git a/pkg/failpoint/fail_test.go b/pkg/failpoint/fail_test.go new file mode 100644 index 000000000..1b79ff6eb --- /dev/null +++ b/pkg/failpoint/fail_test.go @@ -0,0 +1,134 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package failpoint + +import ( + "reflect" + "testing" + "time" +) + +func TestParseTerms(t *testing.T) { + cases := []struct { + terms string + hasError bool + }{ + // off + {"5", true}, + {"*off()", true}, + {"5*off()", true}, + {"5*off(nothing)", true}, + {"5*off(", true}, + {"5*off", false}, + + // error + {"10000error(oops)", true}, + {"10*error(oops)", false}, + {"1234*error(oops))", true}, + {"12342*error()", true}, + + // panic + {"1panic(oops)", true}, + {"1000000*panic(oops)", false}, + {"12345*panic(oops))", true}, + {"12*panic()", true}, + + // delay + {"1*delay(oops)", true}, + {"1000000*delay(-1)", true}, + {"1000000*delay(1)", false}, + + // cascading terms + {"1*delay(1)-", true}, + {"10*delay(2)->", true}, + {"11*delay(3)->10*off(", true}, + {"12*delay(4)->10*of", true}, + {"13*delay(5)->10*off->1000*panic(oops)", false}, + } + + for i, c := range cases { + fp, err := NewFailpoint(t.Name(), c.terms) + + if (err != nil && !c.hasError) || + (err == nil && c.hasError) { + + t.Fatalf("[%v - %s] expected hasError=%v, but got %v", i, c.terms, c.hasError, err) + } + + if err != nil { + continue + } + + if got := fp.Marshal(); !reflect.DeepEqual(got, c.terms) { + t.Fatalf("[%v] expected %v, but got %v", i, c.terms, got) + } + } +} + +func TestEvaluate(t *testing.T) { + terms := "1*error(oops-)->1*off->1*delay(1000)->1*panic(panic)" + + fp, err := NewFailpoint(t.Name(), terms) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + injectedFn := func() error { + if err := fp.Evaluate(); err != nil { + return err + } + return nil + } + + // should return oops- error + if err := injectedFn(); err == nil || err.Error() != "oops-" { + t.Fatalf("expected error %v, but got %v", "oops-", err) + } + + // should return nil + if err := injectedFn(); err != nil { + t.Fatalf("expected nil, but got %v", err) + } + + // should sleep 1s and return nil + now := time.Now() + err = injectedFn() + du := time.Since(now) + if err != nil { + t.Fatalf("expected nil, but got %v", err) + } + if du < 1*time.Second { + t.Fatalf("expected sleep 1s, but got %v", du) + } + + // should panic + defer func() { + if err := recover(); err == nil || err.(string) != "panic" { + t.Fatalf("should panic(panic), but got %v", err) + } + + expected := "0*error(oops-)->0*off->0*delay(1000)->0*panic(panic)" + if got := fp.Marshal(); got != expected { + t.Fatalf("expected %v, but got %v", expected, got) + } + + if err := injectedFn(); err != nil { + t.Fatalf("expected nil, but got %v", err) + } + }() + injectedFn() +} From 822cc51d840775f0c9aa48e6e93cecc657a90f06 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Fri, 17 Jun 2022 00:39:13 +0800 Subject: [PATCH 2/9] runtime/v2: manager supports server interceptor Currently, the runc shimv2 commandline manager doesn't support ttrpc server's customized option, for example, the ttrpc server interceptor. This commit is to allow the task plugin can return the `UnaryServerInterceptor` option to the manager so that the task plugin can do enhancement before handling the incoming request, like API-level failpoint control. Signed-off-by: Wei Fu --- runtime/v2/shim/shim.go | 16 ++++- runtime/v2/shim/shim_darwin.go | 4 +- runtime/v2/shim/shim_freebsd.go | 4 +- runtime/v2/shim/shim_linux.go | 5 +- runtime/v2/shim/shim_windows.go | 2 +- runtime/v2/shim/util.go | 26 +++++++ runtime/v2/shim/util_test.go | 118 ++++++++++++++++++++++++++++++++ 7 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 runtime/v2/shim/util_test.go diff --git a/runtime/v2/shim/shim.go b/runtime/v2/shim/shim.go index 9fa319367..7487ca043 100644 --- a/runtime/v2/shim/shim.go +++ b/runtime/v2/shim/shim.go @@ -108,6 +108,12 @@ type ttrpcService interface { RegisterTTRPC(*ttrpc.Server) error } +type ttrpcServerOptioner interface { + ttrpcService + + UnaryInterceptor() ttrpc.UnaryServerInterceptor +} + type taskService struct { shimapi.TaskService } @@ -370,6 +376,8 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi var ( initialized = plugin.NewPluginSet() ttrpcServices = []ttrpcService{} + + ttrpcUnaryInterceptors = []ttrpc.UnaryServerInterceptor{} ) plugins := plugin.Graph(func(*plugin.Registration) bool { return false }) for _, p := range plugins { @@ -418,10 +426,16 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi if src, ok := instance.(ttrpcService); ok { logrus.WithField("id", id).Debug("registering ttrpc service") ttrpcServices = append(ttrpcServices, src) + + } + + if src, ok := instance.(ttrpcServerOptioner); ok { + ttrpcUnaryInterceptors = append(ttrpcUnaryInterceptors, src.UnaryInterceptor()) } } - server, err := newServer() + unaryInterceptor := chainUnaryServerInterceptors(ttrpcUnaryInterceptors...) + server, err := newServer(ttrpc.WithUnaryServerInterceptor(unaryInterceptor)) if err != nil { return fmt.Errorf("failed creating server: %w", err) } diff --git a/runtime/v2/shim/shim_darwin.go b/runtime/v2/shim/shim_darwin.go index fe833df01..0bdf289bb 100644 --- a/runtime/v2/shim/shim_darwin.go +++ b/runtime/v2/shim/shim_darwin.go @@ -18,8 +18,8 @@ package shim import "github.com/containerd/ttrpc" -func newServer() (*ttrpc.Server, error) { - return ttrpc.NewServer() +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { + return ttrpc.NewServer(opts...) } func subreaper() error { diff --git a/runtime/v2/shim/shim_freebsd.go b/runtime/v2/shim/shim_freebsd.go index fe833df01..0bdf289bb 100644 --- a/runtime/v2/shim/shim_freebsd.go +++ b/runtime/v2/shim/shim_freebsd.go @@ -18,8 +18,8 @@ package shim import "github.com/containerd/ttrpc" -func newServer() (*ttrpc.Server, error) { - return ttrpc.NewServer() +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { + return ttrpc.NewServer(opts...) } func subreaper() error { diff --git a/runtime/v2/shim/shim_linux.go b/runtime/v2/shim/shim_linux.go index 06266a533..1c05c2c56 100644 --- a/runtime/v2/shim/shim_linux.go +++ b/runtime/v2/shim/shim_linux.go @@ -21,8 +21,9 @@ import ( "github.com/containerd/ttrpc" ) -func newServer() (*ttrpc.Server, error) { - return ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser())) +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { + opts = append(opts, ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser())) + return ttrpc.NewServer(opts...) } func subreaper() error { diff --git a/runtime/v2/shim/shim_windows.go b/runtime/v2/shim/shim_windows.go index 4b098ab16..2add7ac33 100644 --- a/runtime/v2/shim/shim_windows.go +++ b/runtime/v2/shim/shim_windows.go @@ -31,7 +31,7 @@ func setupSignals(config Config) (chan os.Signal, error) { return nil, errors.New("not supported") } -func newServer() (*ttrpc.Server, error) { +func newServer(opts ...ttrpc.ServerOpt) (*ttrpc.Server, error) { return nil, errors.New("not supported") } diff --git a/runtime/v2/shim/util.go b/runtime/v2/shim/util.go index bf78fa58f..d1cd47946 100644 --- a/runtime/v2/shim/util.go +++ b/runtime/v2/shim/util.go @@ -30,6 +30,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/protobuf/proto" "github.com/containerd/containerd/protobuf/types" + "github.com/containerd/ttrpc" exec "golang.org/x/sys/execabs" ) @@ -167,3 +168,28 @@ func ReadAddress(path string) (string, error) { } return string(data), nil } + +// chainUnaryServerInterceptors creates a single ttrpc server interceptor from +// a chain of many interceptors executed from first to last. +func chainUnaryServerInterceptors(interceptors ...ttrpc.UnaryServerInterceptor) ttrpc.UnaryServerInterceptor { + n := len(interceptors) + + // force to use default interceptor in ttrpc + if n == 0 { + return nil + } + + return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + currentMethod := method + + for i := n - 1; i > 0; i-- { + interceptor := interceptors[i] + innerMethod := currentMethod + + currentMethod = func(currentCtx context.Context, currentUnmarshal func(interface{}) error) (interface{}, error) { + return interceptor(currentCtx, currentUnmarshal, info, innerMethod) + } + } + return interceptors[0](ctx, unmarshal, info, currentMethod) + } +} diff --git a/runtime/v2/shim/util_test.go b/runtime/v2/shim/util_test.go new file mode 100644 index 000000000..8341bcddb --- /dev/null +++ b/runtime/v2/shim/util_test.go @@ -0,0 +1,118 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shim + +import ( + "context" + "path/filepath" + "reflect" + "testing" + + "github.com/containerd/ttrpc" +) + +func TestChainUnaryServerInterceptors(t *testing.T) { + methodInfo := &ttrpc.UnaryServerInfo{ + FullMethod: filepath.Join("/", t.Name(), "foo"), + } + + type callKey struct{} + callValue := "init" + callCtx := context.WithValue(context.Background(), callKey{}, callValue) + + verifyCallCtxFn := func(ctx context.Context, key interface{}, expected interface{}) { + got := ctx.Value(key) + if !reflect.DeepEqual(expected, got) { + t.Fatalf("[context(key:%s) expected %v, but got %v", key, expected, got) + } + } + + verifyInfoFn := func(info *ttrpc.UnaryServerInfo) { + if !reflect.DeepEqual(methodInfo, info) { + t.Fatalf("[info] expected %+v, but got %+v", methodInfo, info) + } + } + + origUnmarshaler := func(obj interface{}) error { + v := obj.(*int64) + *v *= 2 + return nil + } + + type firstKey struct{} + firstValue := "from first" + var firstUnmarshaler ttrpc.Unmarshaler + first := func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + verifyCallCtxFn(ctx, callKey{}, callValue) + verifyInfoFn(info) + + ctx = context.WithValue(ctx, firstKey{}, firstValue) + + firstUnmarshaler = func(obj interface{}) error { + if err := unmarshal(obj); err != nil { + return err + } + + v := obj.(*int64) + *v *= 2 + return nil + } + + return method(ctx, firstUnmarshaler) + } + + type secondKey struct{} + secondValue := "from second" + second := func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + verifyCallCtxFn(ctx, callKey{}, callValue) + verifyCallCtxFn(ctx, firstKey{}, firstValue) + verifyInfoFn(info) + + v := int64(3) // should return 12 + if err := unmarshal(&v); err != nil { + t.Fatalf("unexpected error %v", err) + } + if expected := int64(12); v != expected { + t.Fatalf("expected int64(%v), but got %v", expected, v) + } + + ctx = context.WithValue(ctx, secondKey{}, secondValue) + return method(ctx, unmarshal) + } + + methodFn := func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + verifyCallCtxFn(ctx, callKey{}, callValue) + verifyCallCtxFn(ctx, firstKey{}, firstValue) + verifyCallCtxFn(ctx, secondKey{}, secondValue) + + v := int64(2) + if err := unmarshal(&v); err != nil { + return nil, err + } + return v, nil + } + + interceptor := chainUnaryServerInterceptors(first, second) + v, err := interceptor(callCtx, origUnmarshaler, methodInfo, methodFn) + if err != nil { + t.Fatalf("expected nil, but got %v", err) + } + + if expected := int64(8); v != expected { + t.Fatalf("expected result is int64(%v), but got %v", expected, v) + } +} From 5f9b318e506b9ebf16fa5caf7a63b47b9ad0b0e5 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sat, 18 Jun 2022 16:15:43 +0800 Subject: [PATCH 3/9] bin/ctr,integration: new runc-shim with failpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added new runc shim binary in integration testing. The shim is named by io.containerd.runc-fp.v1, which allows us to use additional OCI annotation `io.containerd.runtime.v2.shim.failpoint.*` to setup shim task API's failpoint. Since the shim can be shared with multiple container, like what kubernetes pod does, the failpoint will be initialized during setup the shim server. So, the following the container's OCI failpoint's annotation will not work. This commit also updates the ctr tool that we can use `--annotation` to specify annotations when run container. For example: ```bash ➜ ctr run -d --runtime runc-fp.v1 \ --annotation "io.containerd.runtime.v2.shim.failpoint.Kill=1*error(sorry)" \ docker.io/library/alpine:latest testing sleep 1d ➜ ctr t ls TASK PID STATUS testing 147304 RUNNING ➜ ctr t kill -s SIGKILL testing ctr: sorry: unknown ➜ ctr t kill -s SIGKILL testing ➜ sudo ctr t ls TASK PID STATUS testing 147304 STOPPED ``` The runc-fp.v1 shim is based on core runc.v2. We can use it to inject failpoint during testing complicated or big transcation API, like kubernetes PodRunPodsandbox. Signed-off-by: Wei Fu --- Makefile | 6 + cmd/ctr/commands/commands.go | 17 +++ cmd/ctr/commands/run/run_unix.go | 7 + .../cmd/containerd-shim-runc-fp-v1/main.go | 32 ++++ .../cmd/containerd-shim-runc-fp-v1/plugin.go | 144 ++++++++++++++++++ 5 files changed, 206 insertions(+) create mode 100644 integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go create mode 100644 integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go diff --git a/Makefile b/Makefile index 648766c94..099302ff3 100644 --- a/Makefile +++ b/Makefile @@ -223,6 +223,11 @@ cri-integration: binaries bin/cri-integration.test ## run cri integration tests @bash -x ./script/test/cri-integration.sh @rm -rf bin/cri-integration.test +# build runc shimv2 with failpoint control, only used by integration test +bin/containerd-shim-runc-fp-v1: integration/failpoint/cmd/containerd-shim-runc-fp-v1 FORCE + @echo "$(WHALE) $@" + @CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./integration/failpoint/cmd/containerd-shim-runc-fp-v1 + benchmark: ## run benchmarks tests @echo "$(WHALE) $@" @$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root @@ -374,6 +379,7 @@ clean-test: ## clean up debris from previously failed tests @rm -rf /run/containerd/fifo/* @rm -rf /run/containerd-test/* @rm -rf bin/cri-integration.test + @rm -rf bin/containerd-shim-runc-fp-v1 install: ## install binaries @echo "$(WHALE) $@ $(BINARIES)" diff --git a/cmd/ctr/commands/commands.go b/cmd/ctr/commands/commands.go index 37ed75685..4b8b9cf20 100644 --- a/cmd/ctr/commands/commands.go +++ b/cmd/ctr/commands/commands.go @@ -116,6 +116,10 @@ var ( Name: "label", Usage: "specify additional labels (e.g. foo=bar)", }, + cli.StringSliceFlag{ + Name: "annotation", + Usage: "specify additional OCI annotations (e.g. foo=bar)", + }, cli.StringSliceFlag{ Name: "mount", Usage: "specify additional container mount (e.g. type=bind,src=/tmp,dst=/host,options=rbind:ro)", @@ -239,6 +243,19 @@ func LabelArgs(labelStrings []string) map[string]string { return labels } +// AnnotationArgs returns a map of annotation key,value pairs. +func AnnotationArgs(annoStrings []string) (map[string]string, error) { + annotations := make(map[string]string, len(annoStrings)) + for _, anno := range annoStrings { + parts := strings.SplitN(anno, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid key=value format annotation: %v", anno) + } + annotations[parts[0]] = parts[1] + } + return annotations, nil +} + // PrintAsJSON prints input in JSON format func PrintAsJSON(x interface{}) { b, err := json.MarshalIndent(x, "", " ") diff --git a/cmd/ctr/commands/run/run_unix.go b/cmd/ctr/commands/run/run_unix.go index 0ac5fe409..58acc2201 100644 --- a/cmd/ctr/commands/run/run_unix.go +++ b/cmd/ctr/commands/run/run_unix.go @@ -217,6 +217,13 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}), ) } + if annoStrings := context.StringSlice("annotation"); len(annoStrings) > 0 { + annos, err := commands.AnnotationArgs(annoStrings) + if err != nil { + return nil, err + } + opts = append(opts, oci.WithAnnotations(annos)) + } if context.Bool("cni") { cniMeta := &commands.NetworkMetaData{EnableCni: true} diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go new file mode 100644 index 000000000..11d663894 --- /dev/null +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go @@ -0,0 +1,32 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + + "github.com/containerd/containerd/runtime/v2/runc/manager" + _ "github.com/containerd/containerd/runtime/v2/runc/pause" + "github.com/containerd/containerd/runtime/v2/shim" +) + +func main() { + shim.RunManager(context.Background(), manager.NewShimManager("io.containerd.runc-fp.v1")) +} diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go new file mode 100644 index 000000000..7bac1ca95 --- /dev/null +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go @@ -0,0 +1,144 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + taskapi "github.com/containerd/containerd/api/runtime/task/v2" + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/failpoint" + "github.com/containerd/containerd/pkg/shutdown" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/runtime/v2/runc/task" + "github.com/containerd/containerd/runtime/v2/shim" + "github.com/containerd/ttrpc" +) + +const ( + ociConfigFilename = "config.json" + + failpointPrefixKey = "io.containerd.runtime.v2.shim.failpoint." +) + +func init() { + plugin.Register(&plugin.Registration{ + Type: plugin.TTRPCPlugin, + ID: "task", + Requires: []plugin.Type{ + plugin.EventPlugin, + plugin.InternalPlugin, + }, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + pp, err := ic.GetByID(plugin.EventPlugin, "publisher") + if err != nil { + return nil, err + } + ss, err := ic.GetByID(plugin.InternalPlugin, "shutdown") + if err != nil { + return nil, err + } + fps, err := newFailpointFromOCIAnnotation() + if err != nil { + return nil, err + } + service, err := task.NewTaskService(ic.Context, pp.(shim.Publisher), ss.(shutdown.Service)) + if err != nil { + return nil, err + } + + return &taskServiceWithFp{ + fps: fps, + local: service, + }, nil + }, + }) + +} + +type taskServiceWithFp struct { + fps map[string]*failpoint.Failpoint + local taskapi.TaskService +} + +func (s *taskServiceWithFp) RegisterTTRPC(server *ttrpc.Server) error { + taskapi.RegisterTaskService(server, s.local) + return nil +} + +func (s *taskServiceWithFp) UnaryInterceptor() ttrpc.UnaryServerInterceptor { + return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + methodName := filepath.Base(info.FullMethod) + if fp, ok := s.fps[methodName]; ok { + if err := fp.Evaluate(); err != nil { + return nil, err + } + } + return method(ctx, unmarshal) + } +} + +// newFailpointFromOCIAnnotation reloads and parses the annotation from +// bundle-path/config.json. +// +// The annotation controlling task API's failpoint should be like: +// +// io.containerd.runtime.v2.shim.failpoint.Create = 1*off->1*error(please retry) +// +// The `Create` is the shim unary API and the value of annotation is the +// failpoint control. The function will return a set of failpoint controllers. +func newFailpointFromOCIAnnotation() (map[string]*failpoint.Failpoint, error) { + // NOTE: shim's current working dir is in bundle dir. + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current working dir: %w", err) + } + + configPath := filepath.Join(cwd, ociConfigFilename) + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %w", configPath, err) + } + + var spec oci.Spec + if err := json.Unmarshal(data, &spec); err != nil { + return nil, fmt.Errorf("failed to parse oci.Spec(%v): %w", string(data), err) + } + + res := make(map[string]*failpoint.Failpoint) + for k, v := range spec.Annotations { + if !strings.HasPrefix(k, failpointPrefixKey) { + continue + } + + methodName := strings.TrimPrefix(k, failpointPrefixKey) + fp, err := failpoint.NewFailpoint(methodName, v) + if err != nil { + return nil, fmt.Errorf("failed to parse failpoint %v: %w", v, err) + } + res[methodName] = fp + } + return res, nil +} From b297775eafd81e4036845652a722dbfbfb054fae Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sat, 18 Jun 2022 17:06:47 +0800 Subject: [PATCH 4/9] runtime/v2/shim: return if error in load plugin If there is any unskipable error during setuping shim plugins, we should fail return error to prevent from leaky shim instance. For example, there is error during init task plugin, the shim ttrpc server will not contain any shim API method. The any call to the shim will receive that failed to create shim task: service containerd.task.v2.Task: not implemented Then containerd can't use `Shutdown` to let the shim close. The shim will be leaky. And also fail return if there is no ttrpc service. Signed-off-by: Wei Fu --- runtime/v2/shim/shim.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/runtime/v2/shim/shim.go b/runtime/v2/shim/shim.go index 7487ca043..a6078a569 100644 --- a/runtime/v2/shim/shim.go +++ b/runtime/v2/shim/shim.go @@ -417,10 +417,9 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi if err != nil { if plugin.IsSkipPlugin(err) { log.G(ctx).WithError(err).WithField("type", p.Type).Infof("skip loading plugin %q...", id) - } else { - log.G(ctx).WithError(err).Warnf("failed to load plugin %s", id) + continue } - continue + return fmt.Errorf("failed to load plugin %s: %w", id, err) } if src, ok := instance.(ttrpcService); ok { @@ -434,6 +433,10 @@ func run(ctx context.Context, manager Manager, initFunc Init, name string, confi } } + if len(ttrpcServices) == 0 { + return fmt.Errorf("required that ttrpc service") + } + unaryInterceptor := chainUnaryServerInterceptors(ttrpcUnaryInterceptors...) server, err := newServer(ttrpc.WithUnaryServerInterceptor(unaryInterceptor)) if err != nil { From 1ae6e8b076bd2a02cf622b5bf79c17c253dae845 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 19 Jun 2022 19:59:06 +0800 Subject: [PATCH 5/9] pkg/failpoint: add DelegatedEval API Signed-off-by: Wei Fu --- pkg/failpoint/fail.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/failpoint/fail.go b/pkg/failpoint/fail.go index 7056f5c41..e326357c7 100644 --- a/pkg/failpoint/fail.go +++ b/pkg/failpoint/fail.go @@ -25,6 +25,9 @@ import ( "time" ) +// EvalFn is the func type about delegated evaluation. +type EvalFn func() error + // Type is the type of failpoint to specifies which action to take. type Type int @@ -100,6 +103,12 @@ func NewFailpoint(fnName string, terms string) (*Failpoint, error) { // Evaluate evaluates a failpoint. func (fp *Failpoint) Evaluate() error { + fn := fp.DelegatedEval() + return fn() +} + +// DelegatedEval evaluates a failpoint but delegates to caller to fire that. +func (fp *Failpoint) DelegatedEval() EvalFn { var target *failpointEntry func() { @@ -118,9 +127,9 @@ func (fp *Failpoint) Evaluate() error { }() if target == nil { - return nil + return nopEvalFn } - return target.evaluate() + return target.evaluate } // Failpoint returns the current state of control in string format. @@ -291,3 +300,7 @@ func parseInt64(term []byte, terminate byte, val *int64) ([]byte, error) { *val = v return term[i+1:], nil } + +func nopEvalFn() error { + return nil +} From be91a219c2ac5e65c00bbe85c5dff0827d41958b Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 19 Jun 2022 20:23:27 +0800 Subject: [PATCH 6/9] integration: CNI bridge wrapper with failpoint Introduce cni-bridge-fp as CNI bridge plugin wrapper binary for CRI testing. With CNI `io.kubernetes.cri.pod-annotations` capability enabled, the user can inject the failpoint setting by pod's annotation `cniFailpointControlStateDir`, which stores each pod's failpoint setting named by `${K8S_POD_NAMESPACE}-${K8S_POD_NAME}.json`. When the plugin is invoked, the plugin will check the CNI_ARGS to get the failpoint for the CNI_COMMAND from disk. For the testing, the user can prepare setting before RunPodSandbox. Signed-off-by: Wei Fu --- Makefile | 6 + .../failpoint/cmd/cni-bridge-fp/main.go | 209 ++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 integration/failpoint/cmd/cni-bridge-fp/main.go diff --git a/Makefile b/Makefile index 099302ff3..ecf166ac4 100644 --- a/Makefile +++ b/Makefile @@ -228,6 +228,11 @@ bin/containerd-shim-runc-fp-v1: integration/failpoint/cmd/containerd-shim-runc-f @echo "$(WHALE) $@" @CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./integration/failpoint/cmd/containerd-shim-runc-fp-v1 +# build CNI bridge plugin wrapper with failpoint support, only used by integration test +bin/cni-bridge-fp: integration/failpoint/cmd/cni-bridge-fp FORCE + @echo "$(WHALE) $@" + @$(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/cni-bridge-fp + benchmark: ## run benchmarks tests @echo "$(WHALE) $@" @$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root @@ -379,6 +384,7 @@ clean-test: ## clean up debris from previously failed tests @rm -rf /run/containerd/fifo/* @rm -rf /run/containerd-test/* @rm -rf bin/cri-integration.test + @rm -rf bin/cni-bridge-fp @rm -rf bin/containerd-shim-runc-fp-v1 install: ## install binaries diff --git a/integration/failpoint/cmd/cni-bridge-fp/main.go b/integration/failpoint/cmd/cni-bridge-fp/main.go new file mode 100644 index 000000000..2bbba2f3a --- /dev/null +++ b/integration/failpoint/cmd/cni-bridge-fp/main.go @@ -0,0 +1,209 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "github.com/containerd/containerd/pkg/failpoint" + "github.com/containerd/continuity" + "github.com/sirupsen/logrus" +) + +type inheritedPodAnnotations struct { + // CNIFailpointControlStateDir is used to specify the location of + // failpoint control setting. In that such stateDir, the failpoint + // setting is stored in the json file named by + // `${K8S_POD_NAMESPACE}-${K8S_POD_NAME}.json`. The detail of json file + // is described by FailpointConf. + CNIFailpointControlStateDir string `json:"cniFailpointControlStateDir,omitempty"` +} + +// FailpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. +type FailpointConf struct { + Add string `json:"cmdAdd"` + Del string `json:"cmdDel"` + Check string `json:"cmdCheck"` +} + +type netConf struct { + RuntimeConfig struct { + PodAnnotations inheritedPodAnnotations `json:"io.kubernetes.cri.pod-annotations"` + } `json:"runtimeConfig,omitempty"` +} + +func main() { + stdinData, err := ioutil.ReadAll(os.Stdin) + if err != nil { + logrus.Fatalf("failed to read stdin: %v", err) + } + + var conf netConf + if err := json.Unmarshal(stdinData, &conf); err != nil { + logrus.Fatalf("failed to parse network configuration: %v", err) + } + + cniCmd, ok := os.LookupEnv("CNI_COMMAND") + if !ok { + logrus.Fatal("required env CNI_COMMAND") + } + + cniPath, ok := os.LookupEnv("CNI_PATH") + if !ok { + logrus.Fatal("required env CNI_PATH") + } + + evalFn, err := buildFailpointEval(conf.RuntimeConfig.PodAnnotations.CNIFailpointControlStateDir, cniCmd) + if err != nil { + logrus.Fatalf("failed to build failpoint evaluate function: %v", err) + } + + if err := evalFn(); err != nil { + logrus.Fatalf("failpoint: %v", err) + } + + cmd := exec.Command(filepath.Join(cniPath, "bridge")) + cmd.Stdin = bytes.NewReader(stdinData) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + logrus.Fatalf("failed to start bridge cni plugin: %v", err) + } + + if err := cmd.Wait(); err != nil { + logrus.Fatalf("failed to wait for bridge cni plugin: %v", err) + } +} + +// buildFailpointEval will read and update the failpoint setting and then +// return delegated failpoint evaluate function +func buildFailpointEval(stateDir string, cniCmd string) (failpoint.EvalFn, error) { + cniArgs, ok := os.LookupEnv("CNI_ARGS") + if !ok { + return nopEvalFn, nil + } + + target := buildPodFailpointFilepath(stateDir, cniArgs) + if target == "" { + return nopEvalFn, nil + } + + f, err := os.OpenFile(target, os.O_RDWR, 0666) + if err != nil { + if os.IsNotExist(err) { + return nopEvalFn, nil + } + return nil, fmt.Errorf("failed to open file %s: %w", target, err) + } + defer f.Close() + + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + return nil, fmt.Errorf("failed to lock failpoint setting %s: %w", target, err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + data, err := ioutil.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("failed to read failpoint setting %s: %w", target, err) + } + + var conf FailpointConf + if err := json.Unmarshal(data, &conf); err != nil { + return nil, fmt.Errorf("failed to unmarshal failpoint conf %s: %w", string(data), err) + } + + var fpStr *string + switch cniCmd { + case "ADD": + fpStr = &conf.Add + case "DEL": + fpStr = &conf.Del + case "CHECK": + fpStr = &conf.Check + } + + if fpStr == nil || *fpStr == "" { + return nopEvalFn, nil + } + + fp, err := failpoint.NewFailpoint(cniCmd, *fpStr) + if err != nil { + return nil, fmt.Errorf("failed to parse failpoint %s: %w", *fpStr, err) + } + + evalFn := fp.DelegatedEval() + + *fpStr = fp.Marshal() + + data, err = json.Marshal(conf) + if err != nil { + return nil, fmt.Errorf("failed to marshal failpoint conf: %w", err) + } + return evalFn, continuity.AtomicWriteFile(target, data, 0666) +} + +// buildPodFailpointFilepath returns the expected failpoint setting filepath +// by Pod metadata. +func buildPodFailpointFilepath(stateDir, cniArgs string) string { + args := cniArgsIntoKeyValue(cniArgs) + + res := make([]string, 0, 2) + for _, key := range []string{"K8S_POD_NAMESPACE", "K8S_POD_NAME"} { + v, ok := args[key] + if !ok { + break + } + res = append(res, v) + } + if len(res) != 2 { + return "" + } + return filepath.Join(stateDir, strings.Join(res, "-")+".json") +} + +// cniArgsIntoKeyValue converts the CNI ARGS from `key1=value1;key2=value2...` +// into key/value hashmap. +func cniArgsIntoKeyValue(envStr string) map[string]string { + parts := strings.Split(envStr, ";") + res := make(map[string]string, len(parts)) + + for _, part := range parts { + keyValue := strings.SplitN(part, "=", 2) + if len(keyValue) != 2 { + continue + } + + res[keyValue[0]] = keyValue[1] + } + return res +} + +func nopEvalFn() error { + return nil +} From 3c5e80b63e683453fee48093347c7c1e969450a7 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 19 Jun 2022 21:48:10 +0800 Subject: [PATCH 7/9] integration: Add injected failpoint testing for RunPodSandbox Signed-off-by: Wei Fu --- .github/workflows/ci.yml | 4 + integration/main_test.go | 3 +- integration/sandbox_run_rollback_test.go | 122 +++++++++++++++++++++++ script/setup/install-failpoint-binaries | 35 +++++++ script/test/utils.sh | 58 ++++++++++- 5 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 integration/sandbox_run_rollback_test.go create mode 100755 script/setup/install-failpoint-binaries diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22258b2cc..f17b73665 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -425,6 +425,10 @@ jobs: sudo apt-get update sudo apt-get install -y criu + - name: Install failpoint binaries + run: | + script/setup/install-failpoint-binaries + - name: Install containerd env: CGO_ENABLED: 1 diff --git a/integration/main_test.go b/integration/main_test.go index 4659d0f70..18c054b8e 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -174,7 +174,8 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb Uid: util.GenerateID(), Namespace: Randomize(ns), }, - Linux: &runtime.LinuxPodSandboxConfig{}, + Linux: &runtime.LinuxPodSandboxConfig{}, + Annotations: make(map[string]string), } for _, opt := range opts { opt(config) diff --git a/integration/sandbox_run_rollback_test.go b/integration/sandbox_run_rollback_test.go new file mode 100644 index 000000000..fa4502ac8 --- /dev/null +++ b/integration/sandbox_run_rollback_test.go @@ -0,0 +1,122 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package integration + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/pkg/failpoint" + "github.com/stretchr/testify/require" +) + +const ( + failpointRuntimeHandler = "runc-fp" + + failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint." + + failpointCNIStateDirKey = "cniFailpointControlStateDir" +) + +func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + + t.Logf("Inject CNI failpoint") + conf := &failpointConf{ + Add: "1*error(you-shall-not-pass!)", + } + + sbConfig := PodSandboxConfig(t.Name(), "failpoint") + injectCNIFailpoint(t, sbConfig, conf) + + t.Logf("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Equal(t, true, strings.Contains(err.Error(), "you-shall-not-pass!")) + + t.Logf("Retry to create sandbox with same config") + sb, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.NoError(t, err) + + err = runtimeService.StopPodSandbox(sb) + require.NoError(t, err) + + err = runtimeService.RemovePodSandbox(sb) + require.NoError(t, err) +} + +func TestRunPodSandboxWithShimStartFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + + t.Logf("Inject Shim failpoint") + + sbConfig := PodSandboxConfig(t.Name(), "failpoint") + injectShimFailpoint(t, sbConfig, map[string]string{ + "Start": "1*error(no hard feelings)", + }) + + t.Logf("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings")) +} + +// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. +type failpointConf struct { + Add string `json:"cmdAdd"` + Del string `json:"cmdDel"` + Check string `json:"cmdCheck"` +} + +func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf *failpointConf) { + stateDir := t.TempDir() + + metadata := sbConfig.Metadata + fpFilename := filepath.Join(stateDir, + fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name)) + + data, err := json.Marshal(conf) + require.NoError(t, err) + + err = os.WriteFile(fpFilename, data, 0666) + require.NoError(t, err) + + sbConfig.Annotations[failpointCNIStateDirKey] = stateDir +} + +func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) { + for method, fp := range methodFps { + _, err := failpoint.NewFailpoint(method, fp) + require.NoError(t, err, "check failpoint %s for shim method %s", fp, method) + + sbConfig.Annotations[failpointShimPrefixKey+method] = fp + } +} diff --git a/script/setup/install-failpoint-binaries b/script/setup/install-failpoint-binaries new file mode 100755 index 000000000..533eb5423 --- /dev/null +++ b/script/setup/install-failpoint-binaries @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build and install +# +# * cni-bridge-fp into /opt/cni/bin +# * containerd-shim-runc-fp-v1 into /usr/local/bin +# +set -euo pipefail + +base_dir="$(dirname "${BASH_SOURCE[0]}")" +root_dir="$( cd "${base_dir}" && pwd )"/../.. + +cd "${root_dir}" + +CNI_BIN_DIR=${CNI_BIN_DIR:-"/opt/cni/bin"} +make bin/cni-bridge-fp +sudo install bin/cni-bridge-fp "${CNI_BIN_DIR}" + +SHIM_BIN_DIR=${SHIM_BIN_DIR:-"/usr/local/bin"} +make bin/containerd-shim-runc-fp-v1 +sudo install bin/containerd-shim-runc-fp-v1 "${SHIM_BIN_DIR}" diff --git a/script/test/utils.sh b/script/test/utils.sh index fa5e5b49d..595617813 100755 --- a/script/test/utils.sh +++ b/script/test/utils.sh @@ -35,9 +35,10 @@ CONTAINERD_RUNTIME=${CONTAINERD_RUNTIME:-""} if [ -z "${CONTAINERD_CONFIG_FILE}" ]; then config_file="${CONTAINERD_CONFIG_DIR}/containerd-config-cri.toml" truncate --size 0 "${config_file}" + echo "version=2" >> ${config_file} + if command -v sestatus >/dev/null 2>&1; then cat >>${config_file} < Date: Fri, 24 Jun 2022 00:08:26 +0800 Subject: [PATCH 8/9] pkg/failpoint: add FreeBSD link and update pkg doc Signed-off-by: Wei Fu --- pkg/failpoint/fail.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/failpoint/fail.go b/pkg/failpoint/fail.go index e326357c7..d48d65ba8 100644 --- a/pkg/failpoint/fail.go +++ b/pkg/failpoint/fail.go @@ -14,6 +14,10 @@ limitations under the License. */ +// Package failpoint provides the code point in the path, which can be controlled +// by user's variable. +// +// Inspired by FreeBSD fail(9): https://freebsd.org/cgi/man.cgi?query=fail. package failpoint import ( @@ -80,7 +84,7 @@ func (t Type) String() string { // be evaluated. If you want the error injected code should fire in second // call, you can specify "1*off->1*error(oops)". // -// Based on fail(9) freebsd: https://www.freebsd.org/cgi/man.cgi?query=fail&sektion=9&apropos=0&manpath=FreeBSD%2B10.0-RELEASE +// Inspired by FreeBSD fail(9): https://freebsd.org/cgi/man.cgi?query=fail. type Failpoint struct { sync.Mutex From e6a2c079025aa2db5eeb635d4269925d8d3ac0de Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Thu, 21 Jul 2022 23:37:39 +0800 Subject: [PATCH 9/9] integration: simplify CNI-fp and add README.md * Use delegated plugin call to simplify cni-bridge-cni * Add README.md for cni-bridge-cni Signed-off-by: Wei Fu --- go.mod | 2 +- .../failpoint/cmd/cni-bridge-fp/README.md | 159 ++++++++++ .../failpoint/cmd/cni-bridge-fp/main.go | 209 ------------- .../failpoint/cmd/cni-bridge-fp/main_linux.go | 202 ++++++++++++ .../{main.go => main_linux.go} | 3 - .../{plugin.go => plugin_linux.go} | 3 - integration/sandbox_run_rollback_test.go | 4 +- .../containernetworking/cni/pkg/skel/skel.go | 294 ++++++++++++++++++ vendor/modules.txt | 1 + 9 files changed, 659 insertions(+), 218 deletions(-) create mode 100644 integration/failpoint/cmd/cni-bridge-fp/README.md delete mode 100644 integration/failpoint/cmd/cni-bridge-fp/main.go create mode 100644 integration/failpoint/cmd/cni-bridge-fp/main_linux.go rename integration/failpoint/cmd/containerd-shim-runc-fp-v1/{main.go => main_linux.go} (96%) rename integration/failpoint/cmd/containerd-shim-runc-fp-v1/{plugin.go => plugin_linux.go} (99%) create mode 100644 vendor/github.com/containernetworking/cni/pkg/skel/skel.go diff --git a/go.mod b/go.mod index cb96d2f4a..424d6d7ed 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/containerd/ttrpc v1.1.1-0.20220420014843-944ef4a40df3 github.com/containerd/typeurl v1.0.3-0.20220422153119-7f6e6d160d67 github.com/containerd/zfs v1.0.0 + github.com/containernetworking/cni v1.1.1 github.com/containernetworking/plugins v1.1.1 github.com/coreos/go-systemd/v22 v22.3.2 github.com/davecgh/go-spew v1.1.1 @@ -82,7 +83,6 @@ require ( github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cilium/ebpf v0.7.0 // indirect - github.com/containernetworking/cni v1.1.1 // indirect github.com/containers/ocicrypt v1.1.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect diff --git a/integration/failpoint/cmd/cni-bridge-fp/README.md b/integration/failpoint/cmd/cni-bridge-fp/README.md new file mode 100644 index 000000000..065e3b5f0 --- /dev/null +++ b/integration/failpoint/cmd/cni-bridge-fp/README.md @@ -0,0 +1,159 @@ +## cni-bridge-f(ail)p(oint) + +### Overview + +The `cni-bridge-fp` is a CNI plugin which delegates interface-creating function +to [CNI bridge plugin][1] and allows user to inject failpoint before delegation. + +Since the CNI plugin is invoked by binary call from CRI and it is short-lived, +the failpoint need to be configured by a JSON file, which can be persisted. +There is an example about failpoint description. + +```json +{ + "cmdAdd": "1*error(you-shall-not-pass!)->1*panic(again)", + "cmdDel": "1*error(try-again)", + "cmdCheck": "10*off" +} +``` + +* `cmdAdd` (string, optional): The failpoint for `ADD` command. +* `cmdDel` (string, optional): The failpoint for `DEL` command. +* `cmdCheck` (string, optional): The failpoint for `CHECK` command. + +Since the `cmdXXX` can be multiple failpoints, each CNI binary call will update +the current state to make sure the order of execution is expected. + +And the failpoint injection is enabled by pod's annotation. Currently, the key +of customized CNI capabilities in containerd can only be `io.kubernetes.cri.pod-annotations` +and containerd will pass pod's annotations to CNI under the that object. The +user can use the `failpoint.cni.containerd.io/confpath` annotation to enable +failpoint for the pod. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx + annotations: + failpoint.cni.containerd.io/confpath: "/tmp/pod-failpoints.json" +spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +``` + +### Example + +Let's use the following json as failpoint description. + +```bash +$ cat <1*panic(sorry)" +} +EOF +``` + +And use `ip netns` to create persisted net namespace named by `failpoint`. + +```bash +$ sudo ip netns add failpoint +``` + +And then setup the following bash script for demo. + +```bash +$ cat <1*panic(sorry)" +} +``` + +We should setup CNI successfully after retry. When we teardown the interface, +there should be two failpoints. + +```bash +$ sudo CNI_COMMAND=ADD bash /tmp/cni-failpoint-demo-helper.sh +... + +$ sudo CNI_COMMAND=DEL bash /tmp/cni-failpoint-demo-helper.sh +{ + "code": 999, + "msg": "oops" +} + +$ sudo CNI_COMMAND=DEL bash /tmp/cni-failpoint-demo-helper.sh +{ + "code": 999, + "msg": "oops" +} + +$ cat /tmp/cni-failpoint.json | jq . +{ + "cmdAdd": "0*error(try-again)", + "cmdDel": "0*error(oops)", + "cmdCheck": "1*off->1*panic(sorry)" +} +``` + +[1]: diff --git a/integration/failpoint/cmd/cni-bridge-fp/main.go b/integration/failpoint/cmd/cni-bridge-fp/main.go deleted file mode 100644 index 2bbba2f3a..000000000 --- a/integration/failpoint/cmd/cni-bridge-fp/main.go +++ /dev/null @@ -1,209 +0,0 @@ -//go:build linux -// +build linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - "syscall" - - "github.com/containerd/containerd/pkg/failpoint" - "github.com/containerd/continuity" - "github.com/sirupsen/logrus" -) - -type inheritedPodAnnotations struct { - // CNIFailpointControlStateDir is used to specify the location of - // failpoint control setting. In that such stateDir, the failpoint - // setting is stored in the json file named by - // `${K8S_POD_NAMESPACE}-${K8S_POD_NAME}.json`. The detail of json file - // is described by FailpointConf. - CNIFailpointControlStateDir string `json:"cniFailpointControlStateDir,omitempty"` -} - -// FailpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. -type FailpointConf struct { - Add string `json:"cmdAdd"` - Del string `json:"cmdDel"` - Check string `json:"cmdCheck"` -} - -type netConf struct { - RuntimeConfig struct { - PodAnnotations inheritedPodAnnotations `json:"io.kubernetes.cri.pod-annotations"` - } `json:"runtimeConfig,omitempty"` -} - -func main() { - stdinData, err := ioutil.ReadAll(os.Stdin) - if err != nil { - logrus.Fatalf("failed to read stdin: %v", err) - } - - var conf netConf - if err := json.Unmarshal(stdinData, &conf); err != nil { - logrus.Fatalf("failed to parse network configuration: %v", err) - } - - cniCmd, ok := os.LookupEnv("CNI_COMMAND") - if !ok { - logrus.Fatal("required env CNI_COMMAND") - } - - cniPath, ok := os.LookupEnv("CNI_PATH") - if !ok { - logrus.Fatal("required env CNI_PATH") - } - - evalFn, err := buildFailpointEval(conf.RuntimeConfig.PodAnnotations.CNIFailpointControlStateDir, cniCmd) - if err != nil { - logrus.Fatalf("failed to build failpoint evaluate function: %v", err) - } - - if err := evalFn(); err != nil { - logrus.Fatalf("failpoint: %v", err) - } - - cmd := exec.Command(filepath.Join(cniPath, "bridge")) - cmd.Stdin = bytes.NewReader(stdinData) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Start(); err != nil { - logrus.Fatalf("failed to start bridge cni plugin: %v", err) - } - - if err := cmd.Wait(); err != nil { - logrus.Fatalf("failed to wait for bridge cni plugin: %v", err) - } -} - -// buildFailpointEval will read and update the failpoint setting and then -// return delegated failpoint evaluate function -func buildFailpointEval(stateDir string, cniCmd string) (failpoint.EvalFn, error) { - cniArgs, ok := os.LookupEnv("CNI_ARGS") - if !ok { - return nopEvalFn, nil - } - - target := buildPodFailpointFilepath(stateDir, cniArgs) - if target == "" { - return nopEvalFn, nil - } - - f, err := os.OpenFile(target, os.O_RDWR, 0666) - if err != nil { - if os.IsNotExist(err) { - return nopEvalFn, nil - } - return nil, fmt.Errorf("failed to open file %s: %w", target, err) - } - defer f.Close() - - if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { - return nil, fmt.Errorf("failed to lock failpoint setting %s: %w", target, err) - } - defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) - - data, err := ioutil.ReadAll(f) - if err != nil { - return nil, fmt.Errorf("failed to read failpoint setting %s: %w", target, err) - } - - var conf FailpointConf - if err := json.Unmarshal(data, &conf); err != nil { - return nil, fmt.Errorf("failed to unmarshal failpoint conf %s: %w", string(data), err) - } - - var fpStr *string - switch cniCmd { - case "ADD": - fpStr = &conf.Add - case "DEL": - fpStr = &conf.Del - case "CHECK": - fpStr = &conf.Check - } - - if fpStr == nil || *fpStr == "" { - return nopEvalFn, nil - } - - fp, err := failpoint.NewFailpoint(cniCmd, *fpStr) - if err != nil { - return nil, fmt.Errorf("failed to parse failpoint %s: %w", *fpStr, err) - } - - evalFn := fp.DelegatedEval() - - *fpStr = fp.Marshal() - - data, err = json.Marshal(conf) - if err != nil { - return nil, fmt.Errorf("failed to marshal failpoint conf: %w", err) - } - return evalFn, continuity.AtomicWriteFile(target, data, 0666) -} - -// buildPodFailpointFilepath returns the expected failpoint setting filepath -// by Pod metadata. -func buildPodFailpointFilepath(stateDir, cniArgs string) string { - args := cniArgsIntoKeyValue(cniArgs) - - res := make([]string, 0, 2) - for _, key := range []string{"K8S_POD_NAMESPACE", "K8S_POD_NAME"} { - v, ok := args[key] - if !ok { - break - } - res = append(res, v) - } - if len(res) != 2 { - return "" - } - return filepath.Join(stateDir, strings.Join(res, "-")+".json") -} - -// cniArgsIntoKeyValue converts the CNI ARGS from `key1=value1;key2=value2...` -// into key/value hashmap. -func cniArgsIntoKeyValue(envStr string) map[string]string { - parts := strings.Split(envStr, ";") - res := make(map[string]string, len(parts)) - - for _, part := range parts { - keyValue := strings.SplitN(part, "=", 2) - if len(keyValue) != 2 { - continue - } - - res[keyValue[0]] = keyValue[1] - } - return res -} - -func nopEvalFn() error { - return nil -} diff --git a/integration/failpoint/cmd/cni-bridge-fp/main_linux.go b/integration/failpoint/cmd/cni-bridge-fp/main_linux.go new file mode 100644 index 000000000..125c8cf1e --- /dev/null +++ b/integration/failpoint/cmd/cni-bridge-fp/main_linux.go @@ -0,0 +1,202 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "syscall" + + "github.com/containerd/containerd/pkg/failpoint" + "github.com/containerd/continuity" + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" +) + +const delegatedPlugin = "bridge" + +type netConf struct { + RuntimeConfig struct { + PodAnnotations inheritedPodAnnotations `json:"io.kubernetes.cri.pod-annotations"` + } `json:"runtimeConfig,omitempty"` +} + +type inheritedPodAnnotations struct { + // FailpointConfPath represents filepath of failpoint settings. + FailpointConfPath string `json:"failpoint.cni.containerd.io/confpath,omitempty"` +} + +// failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. +type failpointConf struct { + Add string `json:"cmdAdd,omitempty"` + Del string `json:"cmdDel,omitempty"` + Check string `json:"cmdCheck,omitempty"` +} + +func main() { + skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, "bridge with failpoint support") +} + +func cmdAdd(args *skel.CmdArgs) error { + if err := handleFailpoint(args, "ADD"); err != nil { + return err + } + + result, err := invoke.DelegateAdd(context.TODO(), delegatedPlugin, args.StdinData, nil) + if err != nil { + return err + } + return result.Print() +} + +func cmdCheck(args *skel.CmdArgs) error { + if err := handleFailpoint(args, "CHECK"); err != nil { + return err + } + + return invoke.DelegateCheck(context.TODO(), delegatedPlugin, args.StdinData, nil) +} + +func cmdDel(args *skel.CmdArgs) error { + if err := handleFailpoint(args, "DEL"); err != nil { + return err + } + + return invoke.DelegateDel(context.TODO(), delegatedPlugin, args.StdinData, nil) +} + +func handleFailpoint(args *skel.CmdArgs, cmdKind string) error { + var conf netConf + if err := json.Unmarshal(args.StdinData, &conf); err != nil { + return fmt.Errorf("failed to parse network configuration: %w", err) + } + + confPath := conf.RuntimeConfig.PodAnnotations.FailpointConfPath + if len(confPath) == 0 { + return nil + } + + control, err := newFailpointControl(confPath) + if err != nil { + return err + } + + evalFn, err := control.delegatedEvalFn(cmdKind) + if err != nil { + return err + } + return evalFn() +} + +type failpointControl struct { + confPath string +} + +func newFailpointControl(confPath string) (*failpointControl, error) { + if !filepath.IsAbs(confPath) { + return nil, fmt.Errorf("failpoint confPath(%s) is required to be absolute", confPath) + } + + return &failpointControl{ + confPath: confPath, + }, nil +} + +func (c *failpointControl) delegatedEvalFn(cmdKind string) (failpoint.EvalFn, error) { + var resFn failpoint.EvalFn = nopEvalFn + + if err := c.updateTx(func(conf *failpointConf) error { + var fpStr *string + + switch cmdKind { + case "ADD": + fpStr = &conf.Add + case "DEL": + fpStr = &conf.Del + case "CHECK": + fpStr = &conf.Check + } + + if fpStr == nil || *fpStr == "" { + return nil + } + + fp, err := failpoint.NewFailpoint(cmdKind, *fpStr) + if err != nil { + return fmt.Errorf("failed to parse failpoint %s: %w", *fpStr, err) + } + + resFn = fp.DelegatedEval() + + *fpStr = fp.Marshal() + return nil + + }); err != nil { + return nil, err + } + return resFn, nil +} + +func (c *failpointControl) updateTx(updateFn func(conf *failpointConf) error) error { + f, err := os.OpenFile(c.confPath, os.O_RDWR, 0666) + if err != nil { + return fmt.Errorf("failed to open confPath %s: %w", c.confPath, err) + } + defer f.Close() + + if err := flock(f.Fd()); err != nil { + return fmt.Errorf("failed to lock failpoint setting %s: %w", c.confPath, err) + } + defer unflock(f.Fd()) + + data, err := ioutil.ReadAll(f) + if err != nil { + return fmt.Errorf("failed to read failpoint setting %s: %w", c.confPath, err) + } + + var conf failpointConf + if err := json.Unmarshal(data, &conf); err != nil { + return fmt.Errorf("failed to unmarshal failpoint conf %s: %w", string(data), err) + } + + if err := updateFn(&conf); err != nil { + return err + } + + data, err = json.Marshal(conf) + if err != nil { + return fmt.Errorf("failed to marshal failpoint conf: %w", err) + } + return continuity.AtomicWriteFile(c.confPath, data, 0666) +} + +func nopEvalFn() error { + return nil +} + +func flock(fd uintptr) error { + return syscall.Flock(int(fd), syscall.LOCK_EX) +} + +func unflock(fd uintptr) error { + return syscall.Flock(int(fd), syscall.LOCK_UN) +} diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main_linux.go similarity index 96% rename from integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go rename to integration/failpoint/cmd/containerd-shim-runc-fp-v1/main_linux.go index 11d663894..0c6af2526 100644 --- a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main_linux.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - /* Copyright The containerd Authors. diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin_linux.go similarity index 99% rename from integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go rename to integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin_linux.go index 7bac1ca95..1e70c3c2a 100644 --- a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin_linux.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - /* Copyright The containerd Authors. diff --git a/integration/sandbox_run_rollback_test.go b/integration/sandbox_run_rollback_test.go index fa4502ac8..4ab5829b6 100644 --- a/integration/sandbox_run_rollback_test.go +++ b/integration/sandbox_run_rollback_test.go @@ -39,7 +39,7 @@ const ( failpointShimPrefixKey = "io.containerd.runtime.v2.shim.failpoint." - failpointCNIStateDirKey = "cniFailpointControlStateDir" + failpointCNIConfPathKey = "failpoint.cni.containerd.io/confpath" ) func TestRunPodSandboxWithSetupCNIFailure(t *testing.T) { @@ -109,7 +109,7 @@ func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf err = os.WriteFile(fpFilename, data, 0666) require.NoError(t, err) - sbConfig.Annotations[failpointCNIStateDirKey] = stateDir + sbConfig.Annotations[failpointCNIConfPathKey] = fpFilename } func injectShimFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, methodFps map[string]string) { diff --git a/vendor/github.com/containernetworking/cni/pkg/skel/skel.go b/vendor/github.com/containernetworking/cni/pkg/skel/skel.go new file mode 100644 index 000000000..cb8781972 --- /dev/null +++ b/vendor/github.com/containernetworking/cni/pkg/skel/skel.go @@ -0,0 +1,294 @@ +// Copyright 2014-2016 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package skel provides skeleton code for a CNI plugin. +// In particular, it implements argument parsing and validation. +package skel + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strings" + + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/utils" + "github.com/containernetworking/cni/pkg/version" +) + +// CmdArgs captures all the arguments passed in to the plugin +// via both env vars and stdin +type CmdArgs struct { + ContainerID string + Netns string + IfName string + Args string + Path string + StdinData []byte +} + +type dispatcher struct { + Getenv func(string) string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + + ConfVersionDecoder version.ConfigDecoder + VersionReconciler version.Reconciler +} + +type reqForCmdEntry map[string]bool + +func (t *dispatcher) getCmdArgsFromEnv() (string, *CmdArgs, *types.Error) { + var cmd, contID, netns, ifName, args, path string + + vars := []struct { + name string + val *string + reqForCmd reqForCmdEntry + }{ + { + "CNI_COMMAND", + &cmd, + reqForCmdEntry{ + "ADD": true, + "CHECK": true, + "DEL": true, + }, + }, + { + "CNI_CONTAINERID", + &contID, + reqForCmdEntry{ + "ADD": true, + "CHECK": true, + "DEL": true, + }, + }, + { + "CNI_NETNS", + &netns, + reqForCmdEntry{ + "ADD": true, + "CHECK": true, + "DEL": false, + }, + }, + { + "CNI_IFNAME", + &ifName, + reqForCmdEntry{ + "ADD": true, + "CHECK": true, + "DEL": true, + }, + }, + { + "CNI_ARGS", + &args, + reqForCmdEntry{ + "ADD": false, + "CHECK": false, + "DEL": false, + }, + }, + { + "CNI_PATH", + &path, + reqForCmdEntry{ + "ADD": true, + "CHECK": true, + "DEL": true, + }, + }, + } + + argsMissing := make([]string, 0) + for _, v := range vars { + *v.val = t.Getenv(v.name) + if *v.val == "" { + if v.reqForCmd[cmd] || v.name == "CNI_COMMAND" { + argsMissing = append(argsMissing, v.name) + } + } + } + + if len(argsMissing) > 0 { + joined := strings.Join(argsMissing, ",") + return "", nil, types.NewError(types.ErrInvalidEnvironmentVariables, fmt.Sprintf("required env variables [%s] missing", joined), "") + } + + if cmd == "VERSION" { + t.Stdin = bytes.NewReader(nil) + } + + stdinData, err := ioutil.ReadAll(t.Stdin) + if err != nil { + return "", nil, types.NewError(types.ErrIOFailure, fmt.Sprintf("error reading from stdin: %v", err), "") + } + + cmdArgs := &CmdArgs{ + ContainerID: contID, + Netns: netns, + IfName: ifName, + Args: args, + Path: path, + StdinData: stdinData, + } + return cmd, cmdArgs, nil +} + +func (t *dispatcher) checkVersionAndCall(cmdArgs *CmdArgs, pluginVersionInfo version.PluginInfo, toCall func(*CmdArgs) error) *types.Error { + configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData) + if err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } + verErr := t.VersionReconciler.Check(configVersion, pluginVersionInfo) + if verErr != nil { + return types.NewError(types.ErrIncompatibleCNIVersion, "incompatible CNI versions", verErr.Details()) + } + + if err = toCall(cmdArgs); err != nil { + if e, ok := err.(*types.Error); ok { + // don't wrap Error in Error + return e + } + return types.NewError(types.ErrInternal, err.Error(), "") + } + + return nil +} + +func validateConfig(jsonBytes []byte) *types.Error { + var conf struct { + Name string `json:"name"` + } + if err := json.Unmarshal(jsonBytes, &conf); err != nil { + return types.NewError(types.ErrDecodingFailure, fmt.Sprintf("error unmarshall network config: %v", err), "") + } + if conf.Name == "" { + return types.NewError(types.ErrInvalidNetworkConfig, "missing network name", "") + } + if err := utils.ValidateNetworkName(conf.Name); err != nil { + return err + } + return nil +} + +func (t *dispatcher) pluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) *types.Error { + cmd, cmdArgs, err := t.getCmdArgsFromEnv() + if err != nil { + // Print the about string to stderr when no command is set + if err.Code == types.ErrInvalidEnvironmentVariables && t.Getenv("CNI_COMMAND") == "" && about != "" { + _, _ = fmt.Fprintln(t.Stderr, about) + _, _ = fmt.Fprintf(t.Stderr, "CNI protocol versions supported: %s\n", strings.Join(versionInfo.SupportedVersions(), ", ")) + return nil + } + return err + } + + if cmd != "VERSION" { + if err = validateConfig(cmdArgs.StdinData); err != nil { + return err + } + if err = utils.ValidateContainerID(cmdArgs.ContainerID); err != nil { + return err + } + if err = utils.ValidateInterfaceName(cmdArgs.IfName); err != nil { + return err + } + } + + switch cmd { + case "ADD": + err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdAdd) + case "CHECK": + configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData) + if err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } + if gtet, err := version.GreaterThanOrEqualTo(configVersion, "0.4.0"); err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } else if !gtet { + return types.NewError(types.ErrIncompatibleCNIVersion, "config version does not allow CHECK", "") + } + for _, pluginVersion := range versionInfo.SupportedVersions() { + gtet, err := version.GreaterThanOrEqualTo(pluginVersion, configVersion) + if err != nil { + return types.NewError(types.ErrDecodingFailure, err.Error(), "") + } else if gtet { + if err := t.checkVersionAndCall(cmdArgs, versionInfo, cmdCheck); err != nil { + return err + } + return nil + } + } + return types.NewError(types.ErrIncompatibleCNIVersion, "plugin version does not allow CHECK", "") + case "DEL": + err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdDel) + case "VERSION": + if err := versionInfo.Encode(t.Stdout); err != nil { + return types.NewError(types.ErrIOFailure, err.Error(), "") + } + default: + return types.NewError(types.ErrInvalidEnvironmentVariables, fmt.Sprintf("unknown CNI_COMMAND: %v", cmd), "") + } + + return err +} + +// PluginMainWithError is the core "main" for a plugin. It accepts +// callback functions for add, check, and del CNI commands and returns an error. +// +// The caller must also specify what CNI spec versions the plugin supports. +// +// It is the responsibility of the caller to check for non-nil error return. +// +// For a plugin to comply with the CNI spec, it must print any error to stdout +// as JSON and then exit with nonzero status code. +// +// To let this package automatically handle errors and call os.Exit(1) for you, +// use PluginMain() instead. +func PluginMainWithError(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) *types.Error { + return (&dispatcher{ + Getenv: os.Getenv, + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + }).pluginMain(cmdAdd, cmdCheck, cmdDel, versionInfo, about) +} + +// PluginMain is the core "main" for a plugin which includes automatic error handling. +// +// The caller must also specify what CNI spec versions the plugin supports. +// +// The caller can specify an "about" string, which is printed on stderr +// when no CNI_COMMAND is specified. The recommended output is "CNI plugin v" +// +// When an error occurs in either cmdAdd, cmdCheck, or cmdDel, PluginMain will print the error +// as JSON to stdout and call os.Exit(1). +// +// To have more control over error handling, use PluginMainWithError() instead. +func PluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo, about string) { + if e := PluginMainWithError(cmdAdd, cmdCheck, cmdDel, versionInfo, about); e != nil { + if err := e.Print(); err != nil { + log.Print("Error writing error JSON to stdout: ", err) + } + os.Exit(1) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 31808d0ba..ddd7a2c2f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -132,6 +132,7 @@ github.com/containerd/zfs/plugin ## explicit; go 1.14 github.com/containernetworking/cni/libcni github.com/containernetworking/cni/pkg/invoke +github.com/containernetworking/cni/pkg/skel github.com/containernetworking/cni/pkg/types github.com/containernetworking/cni/pkg/types/020 github.com/containernetworking/cni/pkg/types/040