containerd/services/sandbox/controller_service.go
Aditya Ramani 729c97cf39 Handle unexpected shim kill events
When a shim process is unexpectedly killed in a way that was not initiated through containerd - containerd reports the pod as not ready but the containers as running. This results in kubelet repeatedly sending container kill requests that fail since containerd cannot connect to the shim.

Changes:

- In the container exit handler, treat `err: Unavailable` as if the container has already exited out
- When attempting to get a connection to the shim, if the controller isn't available assume that the shim has been killed (needs to be done since we have a separate exit handler that cleans up the reference to the shim controller - before kubelet has the chance to call StopPodSandbox)

Signed-off-by: Aditya Ramani <a_ramani@apple.com>
2023-09-18 12:15:55 -07:00

181 lines
5.7 KiB
Go

/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sandbox
import (
"context"
"google.golang.org/grpc"
"google.golang.org/protobuf/types/known/anypb"
eventtypes "github.com/containerd/containerd/api/events"
api "github.com/containerd/containerd/api/services/sandbox/v1"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/events"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/protobuf"
"github.com/containerd/containerd/sandbox"
)
func init() {
plugin.Register(&plugin.Registration{
Type: plugin.GRPCPlugin,
ID: "sandbox-controllers",
Requires: []plugin.Type{
plugin.SandboxControllerPlugin,
plugin.EventPlugin,
},
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
sc, err := ic.GetByID(plugin.SandboxControllerPlugin, "local")
if err != nil {
return nil, err
}
ep, err := ic.Get(plugin.EventPlugin)
if err != nil {
return nil, err
}
return &controllerService{
local: sc.(sandbox.Controller),
publisher: ep.(events.Publisher),
}, nil
},
})
}
type controllerService struct {
local sandbox.Controller
publisher events.Publisher
api.UnimplementedControllerServer
}
var _ api.ControllerServer = (*controllerService)(nil)
func (s *controllerService) Register(server *grpc.Server) error {
api.RegisterControllerServer(server, s)
return nil
}
func (s *controllerService) Create(ctx context.Context, req *api.ControllerCreateRequest) (*api.ControllerCreateResponse, error) {
log.G(ctx).WithField("req", req).Debug("create sandbox")
// TODO: Rootfs
err := s.local.Create(ctx, req.GetSandboxID(), sandbox.WithOptions(req.GetOptions()))
if err != nil {
return &api.ControllerCreateResponse{}, errdefs.ToGRPC(err)
}
if err := s.publisher.Publish(ctx, "sandboxes/create", &eventtypes.SandboxCreate{
SandboxID: req.GetSandboxID(),
}); err != nil {
return &api.ControllerCreateResponse{}, errdefs.ToGRPC(err)
}
return &api.ControllerCreateResponse{
SandboxID: req.GetSandboxID(),
}, nil
}
func (s *controllerService) Start(ctx context.Context, req *api.ControllerStartRequest) (*api.ControllerStartResponse, error) {
log.G(ctx).WithField("req", req).Debug("start sandbox")
inst, err := s.local.Start(ctx, req.GetSandboxID())
if err != nil {
return &api.ControllerStartResponse{}, errdefs.ToGRPC(err)
}
if err := s.publisher.Publish(ctx, "sandboxes/start", &eventtypes.SandboxStart{
SandboxID: req.GetSandboxID(),
}); err != nil {
return &api.ControllerStartResponse{}, errdefs.ToGRPC(err)
}
return &api.ControllerStartResponse{
SandboxID: inst.SandboxID,
Pid: inst.Pid,
CreatedAt: protobuf.ToTimestamp(inst.CreatedAt),
Labels: inst.Labels,
}, nil
}
func (s *controllerService) Stop(ctx context.Context, req *api.ControllerStopRequest) (*api.ControllerStopResponse, error) {
log.G(ctx).WithField("req", req).Debug("delete sandbox")
return &api.ControllerStopResponse{}, errdefs.ToGRPC(s.local.Stop(ctx, req.GetSandboxID()))
}
func (s *controllerService) Wait(ctx context.Context, req *api.ControllerWaitRequest) (*api.ControllerWaitResponse, error) {
log.G(ctx).WithField("req", req).Debug("wait sandbox")
exitStatus, err := s.local.Wait(ctx, req.GetSandboxID())
if err != nil {
return &api.ControllerWaitResponse{}, errdefs.ToGRPC(err)
}
if err := s.publisher.Publish(ctx, "sandboxes/exit", &eventtypes.SandboxExit{
SandboxID: req.GetSandboxID(),
ExitStatus: exitStatus.ExitStatus,
ExitedAt: protobuf.ToTimestamp(exitStatus.ExitedAt),
}); err != nil {
return &api.ControllerWaitResponse{}, errdefs.ToGRPC(err)
}
return &api.ControllerWaitResponse{
ExitStatus: exitStatus.ExitStatus,
ExitedAt: protobuf.ToTimestamp(exitStatus.ExitedAt),
}, nil
}
func (s *controllerService) Status(ctx context.Context, req *api.ControllerStatusRequest) (*api.ControllerStatusResponse, error) {
log.G(ctx).WithField("req", req).Debug("sandbox status")
cstatus, err := s.local.Status(ctx, req.GetSandboxID(), req.GetVerbose())
if err != nil {
return &api.ControllerStatusResponse{}, errdefs.ToGRPC(err)
}
extra := &anypb.Any{}
if cstatus.Extra != nil {
extra = &anypb.Any{
TypeUrl: cstatus.Extra.GetTypeUrl(),
Value: cstatus.Extra.GetValue(),
}
}
return &api.ControllerStatusResponse{
SandboxID: cstatus.SandboxID,
Pid: cstatus.Pid,
State: cstatus.State,
Info: cstatus.Info,
CreatedAt: protobuf.ToTimestamp(cstatus.CreatedAt),
ExitedAt: protobuf.ToTimestamp(cstatus.ExitedAt),
Extra: extra,
}, nil
}
func (s *controllerService) Shutdown(ctx context.Context, req *api.ControllerShutdownRequest) (*api.ControllerShutdownResponse, error) {
log.G(ctx).WithField("req", req).Debug("shutdown sandbox")
return &api.ControllerShutdownResponse{}, errdefs.ToGRPC(s.local.Shutdown(ctx, req.GetSandboxID()))
}
func (s *controllerService) Metrics(ctx context.Context, req *api.ControllerMetricsRequest) (*api.ControllerMetricsResponse, error) {
log.G(ctx).WithField("req", req).Debug("sandbox metrics")
metrics, err := s.local.Metrics(ctx, req.GetSandboxID())
if err != nil {
return &api.ControllerMetricsResponse{}, errdefs.ToGRPC(err)
}
return &api.ControllerMetricsResponse{
Metrics: metrics,
}, nil
}