Merge pull request #24980 from aveshagarwal/master-project-node-selector

Automatic merge from submit-queue

Add support for admission controller based on namespace node selectors.

This work is to upstream openshift's project node selectors based admission controller.

Fixes https://github.com/kubernetes/kubernetes/issues/17151
This commit is contained in:
Kubernetes Submit Queue
2016-10-14 18:37:33 -07:00
committed by GitHub
5 changed files with 677 additions and 0 deletions

View File

@@ -0,0 +1,216 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podnodeselector
import (
"fmt"
"io"
"reflect"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/admission"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/client/cache"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/controller/informers"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/util/yaml"
)
// The annotation key scheduler.alpha.kubernetes.io/node-selector is for assigning
// node selectors labels to namespaces
var NamespaceNodeSelectors = []string{"scheduler.alpha.kubernetes.io/node-selector"}
func init() {
admission.RegisterPlugin("PodNodeSelector", func(client clientset.Interface, config io.Reader) (admission.Interface, error) {
pluginConfig := readConfig(config)
plugin := NewPodNodeSelector(client, pluginConfig.PodNodeSelectorPluginConfig)
return plugin, nil
})
}
// podNodeSelector is an implementation of admission.Interface.
type podNodeSelector struct {
*admission.Handler
client clientset.Interface
namespaceInformer cache.SharedIndexInformer
// global default node selector and namespace whitelists in a cluster.
clusterNodeSelectors map[string]string
}
type pluginConfig struct {
PodNodeSelectorPluginConfig map[string]string
}
// readConfig reads default value of clusterDefaultNodeSelector
// from the file provided with --admission-control-config-file
// If the file is not supplied, it defaults to ""
// The format in a file:
// podNodeSelectorPluginConfig:
// clusterDefaultNodeSelector: <node-selectors-labels>
// namespace1: <node-selectors-labels>
// namespace2: <node-selectors-labels>
func readConfig(config io.Reader) *pluginConfig {
defaultConfig := &pluginConfig{}
if config == nil || reflect.ValueOf(config).IsNil() {
return defaultConfig
}
d := yaml.NewYAMLOrJSONDecoder(config, 4096)
for {
if err := d.Decode(defaultConfig); err != nil {
if err != io.EOF {
continue
}
}
break
}
return defaultConfig
}
// Admit enforces that pod and its namespace node label selectors matches at least a node in the cluster.
func (p *podNodeSelector) Admit(a admission.Attributes) error {
resource := a.GetResource().GroupResource()
if resource != api.Resource("pods") {
return nil
}
if a.GetSubresource() != "" {
// only run the checks below on pods proper and not subresources
return nil
}
obj := a.GetObject()
pod, ok := obj.(*api.Pod)
if !ok {
glog.Errorf("expected pod but got %s", a.GetKind().Kind)
return nil
}
if !p.WaitForReady() {
return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request"))
}
name := pod.Name
nsName := a.GetNamespace()
var namespace *api.Namespace
namespaceObj, exists, err := p.namespaceInformer.GetStore().Get(&api.Namespace{
ObjectMeta: api.ObjectMeta{
Name: nsName,
Namespace: "",
},
})
if err != nil {
return errors.NewInternalError(err)
}
if exists {
namespace = namespaceObj.(*api.Namespace)
} else {
namespace, err = p.defaultGetNamespace(nsName)
if err != nil {
if errors.IsNotFound(err) {
return err
}
return errors.NewInternalError(err)
}
}
namespaceNodeSelector, err := p.getNodeSelectorMap(namespace)
if err != nil {
return err
}
if labels.Conflicts(namespaceNodeSelector, labels.Set(pod.Spec.NodeSelector)) {
return errors.NewForbidden(resource, name, fmt.Errorf("pod node label selector conflicts with its namespace node label selector"))
}
whitelist, err := labels.ConvertSelectorToLabelsMap(p.clusterNodeSelectors[namespace.Name])
if err != nil {
return err
}
// Merge pod node selector = namespace node selector + current pod node selector
podNodeSelectorLabels := labels.Merge(namespaceNodeSelector, pod.Spec.NodeSelector)
// whitelist verification
if !labels.AreLabelsInWhiteList(podNodeSelectorLabels, whitelist) {
return errors.NewForbidden(resource, name, fmt.Errorf("pod node label selector labels conflict with its namespace whitelist"))
}
// Updated pod node selector = namespace node selector + current pod node selector
pod.Spec.NodeSelector = map[string]string(podNodeSelectorLabels)
return nil
}
func NewPodNodeSelector(client clientset.Interface, clusterNodeSelectors map[string]string) *podNodeSelector {
return &podNodeSelector{
Handler: admission.NewHandler(admission.Create),
client: client,
clusterNodeSelectors: clusterNodeSelectors,
}
}
func (p *podNodeSelector) SetInformerFactory(f informers.SharedInformerFactory) {
p.namespaceInformer = f.Namespaces().Informer()
p.SetReadyFunc(p.namespaceInformer.HasSynced)
}
func (p *podNodeSelector) Validate() error {
if p.namespaceInformer == nil {
return fmt.Errorf("missing namespaceInformer")
}
return nil
}
func (p *podNodeSelector) defaultGetNamespace(name string) (*api.Namespace, error) {
namespace, err := p.client.Core().Namespaces().Get(name)
if err != nil {
return nil, fmt.Errorf("namespace %s does not exist", name)
}
return namespace, nil
}
func (p *podNodeSelector) getNodeSelectorMap(namespace *api.Namespace) (labels.Set, error) {
selector := labels.Set{}
labelsMap := labels.Set{}
var err error
found := false
if len(namespace.ObjectMeta.Annotations) > 0 {
for _, annotation := range NamespaceNodeSelectors {
if ns, ok := namespace.ObjectMeta.Annotations[annotation]; ok {
labelsMap, err = labels.ConvertSelectorToLabelsMap(ns)
if err != nil {
return labels.Set{}, err
}
if labels.Conflicts(selector, labelsMap) {
nsName := namespace.ObjectMeta.Name
return labels.Set{}, fmt.Errorf("%s annotations' node label selectors conflict", nsName)
}
selector = labels.Merge(selector, labelsMap)
found = true
}
}
}
if !found {
selector, err = labels.ConvertSelectorToLabelsMap(p.clusterNodeSelectors["clusterDefaultNodeSelector"])
if err != nil {
return labels.Set{}, err
}
}
return selector, nil
}

View File

@@ -0,0 +1,188 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podnodeselector
import (
"testing"
"time"
"k8s.io/kubernetes/pkg/admission"
"k8s.io/kubernetes/pkg/api"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
"k8s.io/kubernetes/pkg/controller/informers"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/util/wait"
)
// TestPodAdmission verifies various scenarios involving pod/namespace/global node label selectors
func TestPodAdmission(t *testing.T) {
namespace := &api.Namespace{
ObjectMeta: api.ObjectMeta{
Name: "testNamespace",
Namespace: "",
},
}
mockClient := &fake.Clientset{}
handler, informerFactory, err := newHandlerForTest(mockClient)
if err != nil {
t.Errorf("unexpected error initializing handler: %v", err)
}
informerFactory.Start(wait.NeverStop)
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{Name: "testPod", Namespace: "testNamespace"},
}
tests := []struct {
defaultNodeSelector string
namespaceNodeSelector string
whitelist string
podNodeSelector map[string]string
mergedNodeSelector labels.Set
ignoreTestNamespaceNodeSelector bool
admit bool
testName string
}{
{
defaultNodeSelector: "",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{},
ignoreTestNamespaceNodeSelector: true,
admit: true,
testName: "No node selectors",
},
{
defaultNodeSelector: "infra = false",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{"infra": "false"},
ignoreTestNamespaceNodeSelector: true,
admit: true,
testName: "Default node selector and no conflicts",
},
{
defaultNodeSelector: "",
namespaceNodeSelector: " infra = false ",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{"infra": "false"},
admit: true,
testName: "TestNamespace node selector with whitespaces and no conflicts",
},
{
defaultNodeSelector: "infra = false",
namespaceNodeSelector: "infra=true",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{"infra": "true"},
admit: true,
testName: "Default and namespace node selector, no conflicts",
},
{
defaultNodeSelector: "infra = false",
namespaceNodeSelector: "",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{},
admit: true,
testName: "Empty namespace node selector and no conflicts",
},
{
defaultNodeSelector: "infra = false",
namespaceNodeSelector: "infra=true",
podNodeSelector: map[string]string{"env": "test"},
mergedNodeSelector: labels.Set{"infra": "true", "env": "test"},
admit: true,
testName: "TestNamespace and pod node selector, no conflicts",
},
{
defaultNodeSelector: "env = test",
namespaceNodeSelector: "infra=true",
podNodeSelector: map[string]string{"infra": "false"},
admit: false,
testName: "Conflicting pod and namespace node selector, one label",
},
{
defaultNodeSelector: "env=dev",
namespaceNodeSelector: "infra=false, env = test",
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
admit: false,
testName: "Conflicting pod and namespace node selector, multiple labels",
},
{
defaultNodeSelector: "env=dev",
namespaceNodeSelector: "infra=false, env = dev",
whitelist: "env=dev, infra=false, color=blue",
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
mergedNodeSelector: labels.Set{"infra": "false", "env": "dev", "color": "blue"},
admit: true,
testName: "Merged pod node selectors satisfy the whitelist",
},
{
defaultNodeSelector: "env=dev",
namespaceNodeSelector: "infra=false, env = dev",
whitelist: "env=dev, infra=true, color=blue",
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
admit: false,
testName: "Merged pod node selectors conflict with the whitelist",
},
}
for _, test := range tests {
if !test.ignoreTestNamespaceNodeSelector {
namespace.ObjectMeta.Annotations = map[string]string{"scheduler.alpha.kubernetes.io/node-selector": test.namespaceNodeSelector}
handler.namespaceInformer.GetStore().Update(namespace)
}
handler.clusterNodeSelectors = make(map[string]string)
handler.clusterNodeSelectors["clusterDefaultNodeSelector"] = test.defaultNodeSelector
handler.clusterNodeSelectors[namespace.Name] = test.whitelist
pod.Spec = api.PodSpec{NodeSelector: test.podNodeSelector}
err := handler.Admit(admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Create, nil))
if test.admit && err != nil {
t.Errorf("Test: %s, expected no error but got: %s", test.testName, err)
} else if !test.admit && err == nil {
t.Errorf("Test: %s, expected an error", test.testName)
}
if test.admit && !labels.Equals(test.mergedNodeSelector, labels.Set(pod.Spec.NodeSelector)) {
t.Errorf("Test: %s, expected: %s but got: %s", test.testName, test.mergedNodeSelector, pod.Spec.NodeSelector)
}
}
}
func TestHandles(t *testing.T) {
for op, shouldHandle := range map[admission.Operation]bool{
admission.Create: true,
admission.Update: false,
admission.Connect: false,
admission.Delete: false,
} {
nodeEnvionment := NewPodNodeSelector(nil, nil)
if e, a := shouldHandle, nodeEnvionment.Handles(op); e != a {
t.Errorf("%v: shouldHandle=%t, handles=%t", op, e, a)
}
}
}
// newHandlerForTest returns the admission controller configured for testing.
func newHandlerForTest(c clientset.Interface) (*podNodeSelector, informers.SharedInformerFactory, error) {
f := informers.NewSharedInformerFactory(c, 5*time.Minute)
handler := NewPodNodeSelector(c, nil)
plugins := []admission.Interface{handler}
pluginInitializer := admission.NewPluginInitializer(f, nil)
pluginInitializer.Initialize(plugins)
err := admission.Validate(plugins)
return handler, f, err
}