kubernetes/pkg/kubelet/lifecycle/handlers.go
Kubernetes Submit Queue 649c0ddd0e Merge pull request #35342 from timstclair/rejected
Automatic merge from submit-queue

[AppArmor] Hold bad AppArmor pods in pending rather than rejecting

Fixes https://github.com/kubernetes/kubernetes/issues/32837

Overview of the fix:

If the Kubelet needs to reject a Pod for a reason that the control plane doesn't understand (e.g. which AppArmor profiles are installed on the node), then it might contiinuously try to run the pod on the same rejecting node. This change adds a concept of "soft rejection", in which the Pod is admitted, but not allowed to run (and therefore held in a pending state). This prevents the pod from being retried on other nodes, but also prevents the high churn. This is consistent with how other missing local resources (e.g. volumes) is handled.

A side effect of the change is that Pods which are not initially runnable will be retried. This is desired behavior since it avoids a race condition when a new node is brought up but the AppArmor profiles have not yet been loaded on it.

``` release-note
Pods with invalid AppArmor configurations will be held in a Pending state, rather than rejected (failed). Check the pod status message to find out why it is not running.
```

@kubernetes/sig-node @timothysc @rrati @davidopp
2016-11-05 22:52:26 -07:00

167 lines
5.2 KiB
Go

/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package lifecycle
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"strconv"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util/format"
"k8s.io/kubernetes/pkg/security/apparmor"
"k8s.io/kubernetes/pkg/types"
"k8s.io/kubernetes/pkg/util/intstr"
)
type HandlerRunner struct {
httpGetter kubetypes.HttpGetter
commandRunner kubecontainer.ContainerCommandRunner
containerManager podStatusProvider
}
type podStatusProvider interface {
GetPodStatus(uid types.UID, name, namespace string) (*kubecontainer.PodStatus, error)
}
func NewHandlerRunner(httpGetter kubetypes.HttpGetter, commandRunner kubecontainer.ContainerCommandRunner, containerManager podStatusProvider) kubecontainer.HandlerRunner {
return &HandlerRunner{
httpGetter: httpGetter,
commandRunner: commandRunner,
containerManager: containerManager,
}
}
func (hr *HandlerRunner) Run(containerID kubecontainer.ContainerID, pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) {
switch {
case handler.Exec != nil:
var msg string
output, err := hr.commandRunner.RunInContainer(containerID, handler.Exec.Command)
if err != nil {
msg := fmt.Sprintf("Exec lifecycle hook (%v) for Container %q in Pod %q failed - error: %v, message: %q", handler.Exec.Command, container.Name, format.Pod(pod), err, string(output))
glog.V(1).Infof(msg)
}
return msg, err
case handler.HTTPGet != nil:
msg, err := hr.runHTTPHandler(pod, container, handler)
if err != nil {
msg := fmt.Sprintf("Http lifecycle hook (%s) for Container %q in Pod %q failed - error: %v, message: %q", handler.HTTPGet.Path, container.Name, format.Pod(pod), err, msg)
glog.V(1).Infof(msg)
}
return msg, err
default:
err := fmt.Errorf("Invalid handler: %v", handler)
msg := fmt.Sprintf("Cannot run handler: %v", err)
glog.Errorf(msg)
return msg, err
}
}
// resolvePort attempts to turn an IntOrString port reference into a concrete port number.
// If portReference has an int value, it is treated as a literal, and simply returns that value.
// If portReference is a string, an attempt is first made to parse it as an integer. If that fails,
// an attempt is made to find a port with the same name in the container spec.
// If a port with the same name is found, it's ContainerPort value is returned. If no matching
// port is found, an error is returned.
func resolvePort(portReference intstr.IntOrString, container *api.Container) (int, error) {
if portReference.Type == intstr.Int {
return portReference.IntValue(), nil
}
portName := portReference.StrVal
port, err := strconv.Atoi(portName)
if err == nil {
return port, nil
}
for _, portSpec := range container.Ports {
if portSpec.Name == portName {
return int(portSpec.ContainerPort), nil
}
}
return -1, fmt.Errorf("couldn't find port: %v in %v", portReference, container)
}
func (hr *HandlerRunner) runHTTPHandler(pod *api.Pod, container *api.Container, handler *api.Handler) (string, error) {
host := handler.HTTPGet.Host
if len(host) == 0 {
status, err := hr.containerManager.GetPodStatus(pod.UID, pod.Name, pod.Namespace)
if err != nil {
glog.Errorf("Unable to get pod info, event handlers may be invalid.")
return "", err
}
if status.IP == "" {
return "", fmt.Errorf("failed to find networking container: %v", status)
}
host = status.IP
}
var port int
if handler.HTTPGet.Port.Type == intstr.String && len(handler.HTTPGet.Port.StrVal) == 0 {
port = 80
} else {
var err error
port, err = resolvePort(handler.HTTPGet.Port, container)
if err != nil {
return "", err
}
}
url := fmt.Sprintf("http://%s/%s", net.JoinHostPort(host, strconv.Itoa(port)), handler.HTTPGet.Path)
resp, err := hr.httpGetter.Get(url)
return getHttpRespBody(resp), err
}
func getHttpRespBody(resp *http.Response) string {
if resp == nil {
return ""
}
defer resp.Body.Close()
if bytes, err := ioutil.ReadAll(resp.Body); err == nil {
return string(bytes)
}
return ""
}
func NewAppArmorAdmitHandler(validator apparmor.Validator) PodAdmitHandler {
return &appArmorAdmitHandler{
Validator: validator,
}
}
type appArmorAdmitHandler struct {
apparmor.Validator
}
func (a *appArmorAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult {
// If the pod is already running or terminated, no need to recheck AppArmor.
if attrs.Pod.Status.Phase != api.PodPending {
return PodAdmitResult{Admit: true}
}
err := a.Validate(attrs.Pod)
if err == nil {
return PodAdmitResult{Admit: true}
}
return PodAdmitResult{
Admit: false,
Reason: "AppArmor",
Message: fmt.Sprintf("Cannot enforce AppArmor: %v", err),
}
}