Merge pull request #24980 from aveshagarwal/master-project-node-selector

Automatic merge from submit-queue

Add support for admission controller based on namespace node selectors.

This work is to upstream openshift's project node selectors based admission controller.

Fixes https://github.com/kubernetes/kubernetes/issues/17151
This commit is contained in:
Kubernetes Submit Queue 2016-10-14 18:37:33 -07:00 committed by GitHub
commit 19848dddfd
5 changed files with 677 additions and 0 deletions

View File

@ -36,6 +36,7 @@ import (
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/exists"
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/lifecycle"
_ "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label"
_ "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector"
_ "k8s.io/kubernetes/plugin/pkg/admission/resourcequota"
_ "k8s.io/kubernetes/plugin/pkg/admission/security/podsecuritypolicy"
_ "k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny"

View File

@ -17,6 +17,7 @@ limitations under the License.
package labels
import (
"fmt"
"sort"
"strings"
)
@ -78,3 +79,103 @@ func FormatLabels(labelMap map[string]string) string {
}
return l
}
// Conflicts takes 2 maps and returns true if there a key match between
// the maps but the value doesn't match, and returns false in other cases
func Conflicts(labels1, labels2 Set) bool {
small := labels1
big := labels2
if len(labels2) < len(labels1) {
small = labels2
big = labels1
}
for k, v := range small {
if val, match := big[k]; match {
if val != v {
return true
}
}
}
return false
}
// Merge combines given maps, and does not check for any conflicts
// between the maps. In case of conflicts, second map (labels2) wins
func Merge(labels1, labels2 Set) Set {
mergedMap := Set{}
for k, v := range labels1 {
mergedMap[k] = v
}
for k, v := range labels2 {
mergedMap[k] = v
}
return mergedMap
}
// Equals returns true if the given maps are equal
func Equals(labels1, labels2 Set) bool {
if len(labels1) != len(labels2) {
return false
}
for k, v := range labels1 {
value, ok := labels2[k]
if !ok {
return false
}
if value != v {
return false
}
}
return true
}
// AreLabelsInWhiteList verifies if the provided label list
// is in the provided whitelist and returns true, otherwise false.
func AreLabelsInWhiteList(labels, whitelist Set) bool {
if len(whitelist) == 0 {
return true
}
for k, v := range labels {
value, ok := whitelist[k]
if !ok {
return false
}
if value != v {
return false
}
}
return true
}
// ConvertSelectorToLabelsMap converts selector string to labels map
// and validates keys and values
func ConvertSelectorToLabelsMap(selector string) (Set, error) {
labelsMap := Set{}
if len(selector) == 0 {
return labelsMap, nil
}
labels := strings.Split(selector, ",")
for _, label := range labels {
l := strings.Split(label, "=")
if len(l) != 2 {
return labelsMap, fmt.Errorf("invalid selector: %s", l)
}
key := strings.TrimSpace(l[0])
if err := validateLabelKey(key); err != nil {
return labelsMap, err
}
value := strings.TrimSpace(l[1])
if err := validateLabelValue(value); err != nil {
return labelsMap, err
}
labelsMap[key] = value
}
return labelsMap, nil
}

View File

@ -58,3 +58,174 @@ func TestLabelGet(t *testing.T) {
t.Errorf("Set.Get is broken")
}
}
func TestLabelConflict(t *testing.T) {
tests := []struct {
labels1 map[string]string
labels2 map[string]string
conflict bool
}{
{
labels1: map[string]string{},
labels2: map[string]string{},
conflict: false,
},
{
labels1: map[string]string{"env": "test"},
labels2: map[string]string{"infra": "true"},
conflict: false,
},
{
labels1: map[string]string{"env": "test"},
labels2: map[string]string{"infra": "true", "env": "test"},
conflict: false,
},
{
labels1: map[string]string{"env": "test"},
labels2: map[string]string{"env": "dev"},
conflict: true,
},
{
labels1: map[string]string{"env": "test", "infra": "false"},
labels2: map[string]string{"infra": "true", "color": "blue"},
conflict: true,
},
}
for _, test := range tests {
conflict := Conflicts(Set(test.labels1), Set(test.labels2))
if conflict != test.conflict {
t.Errorf("expected: %v but got: %v", test.conflict, conflict)
}
}
}
func TestLabelMerge(t *testing.T) {
tests := []struct {
labels1 map[string]string
labels2 map[string]string
mergedLabels map[string]string
}{
{
labels1: map[string]string{},
labels2: map[string]string{},
mergedLabels: map[string]string{},
},
{
labels1: map[string]string{"infra": "true"},
labels2: map[string]string{},
mergedLabels: map[string]string{"infra": "true"},
},
{
labels1: map[string]string{"infra": "true"},
labels2: map[string]string{"env": "test", "color": "blue"},
mergedLabels: map[string]string{"infra": "true", "env": "test", "color": "blue"},
},
}
for _, test := range tests {
mergedLabels := Merge(Set(test.labels1), Set(test.labels2))
if !Equals(mergedLabels, test.mergedLabels) {
t.Errorf("expected: %v but got: %v", test.mergedLabels, mergedLabels)
}
}
}
func TestLabelSelectorParse(t *testing.T) {
tests := []struct {
selector string
labels map[string]string
valid bool
}{
{
selector: "",
labels: map[string]string{},
valid: true,
},
{
selector: "x=a",
labels: map[string]string{"x": "a"},
valid: true,
},
{
selector: "x=a,y=b,z=c",
labels: map[string]string{"x": "a", "y": "b", "z": "c"},
valid: true,
},
{
selector: " x = a , y = b , z = c ",
labels: map[string]string{"x": "a", "y": "b", "z": "c"},
valid: true,
},
{
selector: "color=green,env=test,service=front",
labels: map[string]string{"color": "green", "env": "test", "service": "front"},
valid: true,
},
{
selector: "color=green, env=test, service=front",
labels: map[string]string{"color": "green", "env": "test", "service": "front"},
valid: true,
},
{
selector: ",",
labels: map[string]string{},
valid: false,
},
{
selector: "x",
labels: map[string]string{},
valid: false,
},
{
selector: "x,y",
labels: map[string]string{},
valid: false,
},
{
selector: "x=$y",
labels: map[string]string{},
valid: false,
},
{
selector: "x!=y",
labels: map[string]string{},
valid: false,
},
{
selector: "x==y",
labels: map[string]string{},
valid: false,
},
{
selector: "x=a||y=b",
labels: map[string]string{},
valid: false,
},
{
selector: "x in (y)",
labels: map[string]string{},
valid: false,
},
{
selector: "x notin (y)",
labels: map[string]string{},
valid: false,
},
{
selector: "x y",
labels: map[string]string{},
valid: false,
},
}
for _, test := range tests {
labels, err := ConvertSelectorToLabelsMap(test.selector)
if test.valid && err != nil {
t.Errorf("selector: %s, expected no error but got: %s", test.selector, err)
} else if !test.valid && err == nil {
t.Errorf("selector: %s, expected an error", test.selector)
}
if !Equals(Set(labels), test.labels) {
t.Errorf("expected: %s but got: %s", test.labels, labels)
}
}
}

View File

@ -0,0 +1,216 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podnodeselector
import (
"fmt"
"io"
"reflect"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/admission"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/client/cache"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/controller/informers"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/util/yaml"
)
// The annotation key scheduler.alpha.kubernetes.io/node-selector is for assigning
// node selectors labels to namespaces
var NamespaceNodeSelectors = []string{"scheduler.alpha.kubernetes.io/node-selector"}
func init() {
admission.RegisterPlugin("PodNodeSelector", func(client clientset.Interface, config io.Reader) (admission.Interface, error) {
pluginConfig := readConfig(config)
plugin := NewPodNodeSelector(client, pluginConfig.PodNodeSelectorPluginConfig)
return plugin, nil
})
}
// podNodeSelector is an implementation of admission.Interface.
type podNodeSelector struct {
*admission.Handler
client clientset.Interface
namespaceInformer cache.SharedIndexInformer
// global default node selector and namespace whitelists in a cluster.
clusterNodeSelectors map[string]string
}
type pluginConfig struct {
PodNodeSelectorPluginConfig map[string]string
}
// readConfig reads default value of clusterDefaultNodeSelector
// from the file provided with --admission-control-config-file
// If the file is not supplied, it defaults to ""
// The format in a file:
// podNodeSelectorPluginConfig:
// clusterDefaultNodeSelector: <node-selectors-labels>
// namespace1: <node-selectors-labels>
// namespace2: <node-selectors-labels>
func readConfig(config io.Reader) *pluginConfig {
defaultConfig := &pluginConfig{}
if config == nil || reflect.ValueOf(config).IsNil() {
return defaultConfig
}
d := yaml.NewYAMLOrJSONDecoder(config, 4096)
for {
if err := d.Decode(defaultConfig); err != nil {
if err != io.EOF {
continue
}
}
break
}
return defaultConfig
}
// Admit enforces that pod and its namespace node label selectors matches at least a node in the cluster.
func (p *podNodeSelector) Admit(a admission.Attributes) error {
resource := a.GetResource().GroupResource()
if resource != api.Resource("pods") {
return nil
}
if a.GetSubresource() != "" {
// only run the checks below on pods proper and not subresources
return nil
}
obj := a.GetObject()
pod, ok := obj.(*api.Pod)
if !ok {
glog.Errorf("expected pod but got %s", a.GetKind().Kind)
return nil
}
if !p.WaitForReady() {
return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request"))
}
name := pod.Name
nsName := a.GetNamespace()
var namespace *api.Namespace
namespaceObj, exists, err := p.namespaceInformer.GetStore().Get(&api.Namespace{
ObjectMeta: api.ObjectMeta{
Name: nsName,
Namespace: "",
},
})
if err != nil {
return errors.NewInternalError(err)
}
if exists {
namespace = namespaceObj.(*api.Namespace)
} else {
namespace, err = p.defaultGetNamespace(nsName)
if err != nil {
if errors.IsNotFound(err) {
return err
}
return errors.NewInternalError(err)
}
}
namespaceNodeSelector, err := p.getNodeSelectorMap(namespace)
if err != nil {
return err
}
if labels.Conflicts(namespaceNodeSelector, labels.Set(pod.Spec.NodeSelector)) {
return errors.NewForbidden(resource, name, fmt.Errorf("pod node label selector conflicts with its namespace node label selector"))
}
whitelist, err := labels.ConvertSelectorToLabelsMap(p.clusterNodeSelectors[namespace.Name])
if err != nil {
return err
}
// Merge pod node selector = namespace node selector + current pod node selector
podNodeSelectorLabels := labels.Merge(namespaceNodeSelector, pod.Spec.NodeSelector)
// whitelist verification
if !labels.AreLabelsInWhiteList(podNodeSelectorLabels, whitelist) {
return errors.NewForbidden(resource, name, fmt.Errorf("pod node label selector labels conflict with its namespace whitelist"))
}
// Updated pod node selector = namespace node selector + current pod node selector
pod.Spec.NodeSelector = map[string]string(podNodeSelectorLabels)
return nil
}
func NewPodNodeSelector(client clientset.Interface, clusterNodeSelectors map[string]string) *podNodeSelector {
return &podNodeSelector{
Handler: admission.NewHandler(admission.Create),
client: client,
clusterNodeSelectors: clusterNodeSelectors,
}
}
func (p *podNodeSelector) SetInformerFactory(f informers.SharedInformerFactory) {
p.namespaceInformer = f.Namespaces().Informer()
p.SetReadyFunc(p.namespaceInformer.HasSynced)
}
func (p *podNodeSelector) Validate() error {
if p.namespaceInformer == nil {
return fmt.Errorf("missing namespaceInformer")
}
return nil
}
func (p *podNodeSelector) defaultGetNamespace(name string) (*api.Namespace, error) {
namespace, err := p.client.Core().Namespaces().Get(name)
if err != nil {
return nil, fmt.Errorf("namespace %s does not exist", name)
}
return namespace, nil
}
func (p *podNodeSelector) getNodeSelectorMap(namespace *api.Namespace) (labels.Set, error) {
selector := labels.Set{}
labelsMap := labels.Set{}
var err error
found := false
if len(namespace.ObjectMeta.Annotations) > 0 {
for _, annotation := range NamespaceNodeSelectors {
if ns, ok := namespace.ObjectMeta.Annotations[annotation]; ok {
labelsMap, err = labels.ConvertSelectorToLabelsMap(ns)
if err != nil {
return labels.Set{}, err
}
if labels.Conflicts(selector, labelsMap) {
nsName := namespace.ObjectMeta.Name
return labels.Set{}, fmt.Errorf("%s annotations' node label selectors conflict", nsName)
}
selector = labels.Merge(selector, labelsMap)
found = true
}
}
}
if !found {
selector, err = labels.ConvertSelectorToLabelsMap(p.clusterNodeSelectors["clusterDefaultNodeSelector"])
if err != nil {
return labels.Set{}, err
}
}
return selector, nil
}

View File

@ -0,0 +1,188 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podnodeselector
import (
"testing"
"time"
"k8s.io/kubernetes/pkg/admission"
"k8s.io/kubernetes/pkg/api"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
"k8s.io/kubernetes/pkg/controller/informers"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/util/wait"
)
// TestPodAdmission verifies various scenarios involving pod/namespace/global node label selectors
func TestPodAdmission(t *testing.T) {
namespace := &api.Namespace{
ObjectMeta: api.ObjectMeta{
Name: "testNamespace",
Namespace: "",
},
}
mockClient := &fake.Clientset{}
handler, informerFactory, err := newHandlerForTest(mockClient)
if err != nil {
t.Errorf("unexpected error initializing handler: %v", err)
}
informerFactory.Start(wait.NeverStop)
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{Name: "testPod", Namespace: "testNamespace"},
}
tests := []struct {
defaultNodeSelector string
namespaceNodeSelector string
whitelist string
podNodeSelector map[string]string
mergedNodeSelector labels.Set
ignoreTestNamespaceNodeSelector bool
admit bool
testName string
}{
{
defaultNodeSelector: "",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{},
ignoreTestNamespaceNodeSelector: true,
admit: true,
testName: "No node selectors",
},
{
defaultNodeSelector: "infra = false",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{"infra": "false"},
ignoreTestNamespaceNodeSelector: true,
admit: true,
testName: "Default node selector and no conflicts",
},
{
defaultNodeSelector: "",
namespaceNodeSelector: " infra = false ",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{"infra": "false"},
admit: true,
testName: "TestNamespace node selector with whitespaces and no conflicts",
},
{
defaultNodeSelector: "infra = false",
namespaceNodeSelector: "infra=true",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{"infra": "true"},
admit: true,
testName: "Default and namespace node selector, no conflicts",
},
{
defaultNodeSelector: "infra = false",
namespaceNodeSelector: "",
podNodeSelector: map[string]string{},
mergedNodeSelector: labels.Set{},
admit: true,
testName: "Empty namespace node selector and no conflicts",
},
{
defaultNodeSelector: "infra = false",
namespaceNodeSelector: "infra=true",
podNodeSelector: map[string]string{"env": "test"},
mergedNodeSelector: labels.Set{"infra": "true", "env": "test"},
admit: true,
testName: "TestNamespace and pod node selector, no conflicts",
},
{
defaultNodeSelector: "env = test",
namespaceNodeSelector: "infra=true",
podNodeSelector: map[string]string{"infra": "false"},
admit: false,
testName: "Conflicting pod and namespace node selector, one label",
},
{
defaultNodeSelector: "env=dev",
namespaceNodeSelector: "infra=false, env = test",
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
admit: false,
testName: "Conflicting pod and namespace node selector, multiple labels",
},
{
defaultNodeSelector: "env=dev",
namespaceNodeSelector: "infra=false, env = dev",
whitelist: "env=dev, infra=false, color=blue",
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
mergedNodeSelector: labels.Set{"infra": "false", "env": "dev", "color": "blue"},
admit: true,
testName: "Merged pod node selectors satisfy the whitelist",
},
{
defaultNodeSelector: "env=dev",
namespaceNodeSelector: "infra=false, env = dev",
whitelist: "env=dev, infra=true, color=blue",
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
admit: false,
testName: "Merged pod node selectors conflict with the whitelist",
},
}
for _, test := range tests {
if !test.ignoreTestNamespaceNodeSelector {
namespace.ObjectMeta.Annotations = map[string]string{"scheduler.alpha.kubernetes.io/node-selector": test.namespaceNodeSelector}
handler.namespaceInformer.GetStore().Update(namespace)
}
handler.clusterNodeSelectors = make(map[string]string)
handler.clusterNodeSelectors["clusterDefaultNodeSelector"] = test.defaultNodeSelector
handler.clusterNodeSelectors[namespace.Name] = test.whitelist
pod.Spec = api.PodSpec{NodeSelector: test.podNodeSelector}
err := handler.Admit(admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Create, nil))
if test.admit && err != nil {
t.Errorf("Test: %s, expected no error but got: %s", test.testName, err)
} else if !test.admit && err == nil {
t.Errorf("Test: %s, expected an error", test.testName)
}
if test.admit && !labels.Equals(test.mergedNodeSelector, labels.Set(pod.Spec.NodeSelector)) {
t.Errorf("Test: %s, expected: %s but got: %s", test.testName, test.mergedNodeSelector, pod.Spec.NodeSelector)
}
}
}
func TestHandles(t *testing.T) {
for op, shouldHandle := range map[admission.Operation]bool{
admission.Create: true,
admission.Update: false,
admission.Connect: false,
admission.Delete: false,
} {
nodeEnvionment := NewPodNodeSelector(nil, nil)
if e, a := shouldHandle, nodeEnvionment.Handles(op); e != a {
t.Errorf("%v: shouldHandle=%t, handles=%t", op, e, a)
}
}
}
// newHandlerForTest returns the admission controller configured for testing.
func newHandlerForTest(c clientset.Interface) (*podNodeSelector, informers.SharedInformerFactory, error) {
f := informers.NewSharedInformerFactory(c, 5*time.Minute)
handler := NewPodNodeSelector(c, nil)
plugins := []admission.Interface{handler}
pluginInitializer := admission.NewPluginInitializer(f, nil)
pluginInitializer.Initialize(plugins)
err := admission.Validate(plugins)
return handler, f, err
}