diff --git a/Makefile b/Makefile index 648766c94..099302ff3 100644 --- a/Makefile +++ b/Makefile @@ -223,6 +223,11 @@ cri-integration: binaries bin/cri-integration.test ## run cri integration tests @bash -x ./script/test/cri-integration.sh @rm -rf bin/cri-integration.test +# build runc shimv2 with failpoint control, only used by integration test +bin/containerd-shim-runc-fp-v1: integration/failpoint/cmd/containerd-shim-runc-fp-v1 FORCE + @echo "$(WHALE) $@" + @CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./integration/failpoint/cmd/containerd-shim-runc-fp-v1 + benchmark: ## run benchmarks tests @echo "$(WHALE) $@" @$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root @@ -374,6 +379,7 @@ clean-test: ## clean up debris from previously failed tests @rm -rf /run/containerd/fifo/* @rm -rf /run/containerd-test/* @rm -rf bin/cri-integration.test + @rm -rf bin/containerd-shim-runc-fp-v1 install: ## install binaries @echo "$(WHALE) $@ $(BINARIES)" diff --git a/cmd/ctr/commands/commands.go b/cmd/ctr/commands/commands.go index 37ed75685..4b8b9cf20 100644 --- a/cmd/ctr/commands/commands.go +++ b/cmd/ctr/commands/commands.go @@ -116,6 +116,10 @@ var ( Name: "label", Usage: "specify additional labels (e.g. foo=bar)", }, + cli.StringSliceFlag{ + Name: "annotation", + Usage: "specify additional OCI annotations (e.g. foo=bar)", + }, cli.StringSliceFlag{ Name: "mount", Usage: "specify additional container mount (e.g. type=bind,src=/tmp,dst=/host,options=rbind:ro)", @@ -239,6 +243,19 @@ func LabelArgs(labelStrings []string) map[string]string { return labels } +// AnnotationArgs returns a map of annotation key,value pairs. +func AnnotationArgs(annoStrings []string) (map[string]string, error) { + annotations := make(map[string]string, len(annoStrings)) + for _, anno := range annoStrings { + parts := strings.SplitN(anno, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid key=value format annotation: %v", anno) + } + annotations[parts[0]] = parts[1] + } + return annotations, nil +} + // PrintAsJSON prints input in JSON format func PrintAsJSON(x interface{}) { b, err := json.MarshalIndent(x, "", " ") diff --git a/cmd/ctr/commands/run/run_unix.go b/cmd/ctr/commands/run/run_unix.go index 0ac5fe409..58acc2201 100644 --- a/cmd/ctr/commands/run/run_unix.go +++ b/cmd/ctr/commands/run/run_unix.go @@ -217,6 +217,13 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}), ) } + if annoStrings := context.StringSlice("annotation"); len(annoStrings) > 0 { + annos, err := commands.AnnotationArgs(annoStrings) + if err != nil { + return nil, err + } + opts = append(opts, oci.WithAnnotations(annos)) + } if context.Bool("cni") { cniMeta := &commands.NetworkMetaData{EnableCni: true} diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go new file mode 100644 index 000000000..11d663894 --- /dev/null +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/main.go @@ -0,0 +1,32 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + + "github.com/containerd/containerd/runtime/v2/runc/manager" + _ "github.com/containerd/containerd/runtime/v2/runc/pause" + "github.com/containerd/containerd/runtime/v2/shim" +) + +func main() { + shim.RunManager(context.Background(), manager.NewShimManager("io.containerd.runc-fp.v1")) +} diff --git a/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go new file mode 100644 index 000000000..7bac1ca95 --- /dev/null +++ b/integration/failpoint/cmd/containerd-shim-runc-fp-v1/plugin.go @@ -0,0 +1,144 @@ +//go:build linux +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + taskapi "github.com/containerd/containerd/api/runtime/task/v2" + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/failpoint" + "github.com/containerd/containerd/pkg/shutdown" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/runtime/v2/runc/task" + "github.com/containerd/containerd/runtime/v2/shim" + "github.com/containerd/ttrpc" +) + +const ( + ociConfigFilename = "config.json" + + failpointPrefixKey = "io.containerd.runtime.v2.shim.failpoint." +) + +func init() { + plugin.Register(&plugin.Registration{ + Type: plugin.TTRPCPlugin, + ID: "task", + Requires: []plugin.Type{ + plugin.EventPlugin, + plugin.InternalPlugin, + }, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + pp, err := ic.GetByID(plugin.EventPlugin, "publisher") + if err != nil { + return nil, err + } + ss, err := ic.GetByID(plugin.InternalPlugin, "shutdown") + if err != nil { + return nil, err + } + fps, err := newFailpointFromOCIAnnotation() + if err != nil { + return nil, err + } + service, err := task.NewTaskService(ic.Context, pp.(shim.Publisher), ss.(shutdown.Service)) + if err != nil { + return nil, err + } + + return &taskServiceWithFp{ + fps: fps, + local: service, + }, nil + }, + }) + +} + +type taskServiceWithFp struct { + fps map[string]*failpoint.Failpoint + local taskapi.TaskService +} + +func (s *taskServiceWithFp) RegisterTTRPC(server *ttrpc.Server) error { + taskapi.RegisterTaskService(server, s.local) + return nil +} + +func (s *taskServiceWithFp) UnaryInterceptor() ttrpc.UnaryServerInterceptor { + return func(ctx context.Context, unmarshal ttrpc.Unmarshaler, info *ttrpc.UnaryServerInfo, method ttrpc.Method) (interface{}, error) { + methodName := filepath.Base(info.FullMethod) + if fp, ok := s.fps[methodName]; ok { + if err := fp.Evaluate(); err != nil { + return nil, err + } + } + return method(ctx, unmarshal) + } +} + +// newFailpointFromOCIAnnotation reloads and parses the annotation from +// bundle-path/config.json. +// +// The annotation controlling task API's failpoint should be like: +// +// io.containerd.runtime.v2.shim.failpoint.Create = 1*off->1*error(please retry) +// +// The `Create` is the shim unary API and the value of annotation is the +// failpoint control. The function will return a set of failpoint controllers. +func newFailpointFromOCIAnnotation() (map[string]*failpoint.Failpoint, error) { + // NOTE: shim's current working dir is in bundle dir. + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current working dir: %w", err) + } + + configPath := filepath.Join(cwd, ociConfigFilename) + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %w", configPath, err) + } + + var spec oci.Spec + if err := json.Unmarshal(data, &spec); err != nil { + return nil, fmt.Errorf("failed to parse oci.Spec(%v): %w", string(data), err) + } + + res := make(map[string]*failpoint.Failpoint) + for k, v := range spec.Annotations { + if !strings.HasPrefix(k, failpointPrefixKey) { + continue + } + + methodName := strings.TrimPrefix(k, failpointPrefixKey) + fp, err := failpoint.NewFailpoint(methodName, v) + if err != nil { + return nil, fmt.Errorf("failed to parse failpoint %v: %w", v, err) + } + res[methodName] = fp + } + return res, nil +}