Merge pull request #24980 from aveshagarwal/master-project-node-selector
Automatic merge from submit-queue Add support for admission controller based on namespace node selectors. This work is to upstream openshift's project node selectors based admission controller. Fixes https://github.com/kubernetes/kubernetes/issues/17151
This commit is contained in:
Kubernetes Submit Queue
216
plugin/pkg/admission/podnodeselector/admission.go
Normal file
216
plugin/pkg/admission/podnodeselector/admission.go
Normal file
@@ -0,0 +1,216 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package podnodeselector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
"k8s.io/kubernetes/pkg/admission"
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
"k8s.io/kubernetes/pkg/api/errors"
|
||||
"k8s.io/kubernetes/pkg/client/cache"
|
||||
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
|
||||
"k8s.io/kubernetes/pkg/controller/informers"
|
||||
"k8s.io/kubernetes/pkg/labels"
|
||||
"k8s.io/kubernetes/pkg/util/yaml"
|
||||
)
|
||||
|
||||
// The annotation key scheduler.alpha.kubernetes.io/node-selector is for assigning
|
||||
// node selectors labels to namespaces
|
||||
var NamespaceNodeSelectors = []string{"scheduler.alpha.kubernetes.io/node-selector"}
|
||||
|
||||
func init() {
|
||||
admission.RegisterPlugin("PodNodeSelector", func(client clientset.Interface, config io.Reader) (admission.Interface, error) {
|
||||
pluginConfig := readConfig(config)
|
||||
plugin := NewPodNodeSelector(client, pluginConfig.PodNodeSelectorPluginConfig)
|
||||
return plugin, nil
|
||||
})
|
||||
}
|
||||
|
||||
// podNodeSelector is an implementation of admission.Interface.
|
||||
type podNodeSelector struct {
|
||||
*admission.Handler
|
||||
client clientset.Interface
|
||||
namespaceInformer cache.SharedIndexInformer
|
||||
// global default node selector and namespace whitelists in a cluster.
|
||||
clusterNodeSelectors map[string]string
|
||||
}
|
||||
|
||||
type pluginConfig struct {
|
||||
PodNodeSelectorPluginConfig map[string]string
|
||||
}
|
||||
|
||||
// readConfig reads default value of clusterDefaultNodeSelector
|
||||
// from the file provided with --admission-control-config-file
|
||||
// If the file is not supplied, it defaults to ""
|
||||
// The format in a file:
|
||||
// podNodeSelectorPluginConfig:
|
||||
// clusterDefaultNodeSelector: <node-selectors-labels>
|
||||
// namespace1: <node-selectors-labels>
|
||||
// namespace2: <node-selectors-labels>
|
||||
func readConfig(config io.Reader) *pluginConfig {
|
||||
defaultConfig := &pluginConfig{}
|
||||
if config == nil || reflect.ValueOf(config).IsNil() {
|
||||
return defaultConfig
|
||||
}
|
||||
d := yaml.NewYAMLOrJSONDecoder(config, 4096)
|
||||
for {
|
||||
if err := d.Decode(defaultConfig); err != nil {
|
||||
if err != io.EOF {
|
||||
continue
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return defaultConfig
|
||||
}
|
||||
|
||||
// Admit enforces that pod and its namespace node label selectors matches at least a node in the cluster.
|
||||
func (p *podNodeSelector) Admit(a admission.Attributes) error {
|
||||
resource := a.GetResource().GroupResource()
|
||||
if resource != api.Resource("pods") {
|
||||
return nil
|
||||
}
|
||||
if a.GetSubresource() != "" {
|
||||
// only run the checks below on pods proper and not subresources
|
||||
return nil
|
||||
}
|
||||
|
||||
obj := a.GetObject()
|
||||
pod, ok := obj.(*api.Pod)
|
||||
if !ok {
|
||||
glog.Errorf("expected pod but got %s", a.GetKind().Kind)
|
||||
return nil
|
||||
}
|
||||
|
||||
if !p.WaitForReady() {
|
||||
return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request"))
|
||||
}
|
||||
|
||||
name := pod.Name
|
||||
nsName := a.GetNamespace()
|
||||
var namespace *api.Namespace
|
||||
|
||||
namespaceObj, exists, err := p.namespaceInformer.GetStore().Get(&api.Namespace{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Name: nsName,
|
||||
Namespace: "",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return errors.NewInternalError(err)
|
||||
}
|
||||
if exists {
|
||||
namespace = namespaceObj.(*api.Namespace)
|
||||
} else {
|
||||
namespace, err = p.defaultGetNamespace(nsName)
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
return errors.NewInternalError(err)
|
||||
}
|
||||
}
|
||||
namespaceNodeSelector, err := p.getNodeSelectorMap(namespace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if labels.Conflicts(namespaceNodeSelector, labels.Set(pod.Spec.NodeSelector)) {
|
||||
return errors.NewForbidden(resource, name, fmt.Errorf("pod node label selector conflicts with its namespace node label selector"))
|
||||
}
|
||||
|
||||
whitelist, err := labels.ConvertSelectorToLabelsMap(p.clusterNodeSelectors[namespace.Name])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Merge pod node selector = namespace node selector + current pod node selector
|
||||
podNodeSelectorLabels := labels.Merge(namespaceNodeSelector, pod.Spec.NodeSelector)
|
||||
|
||||
// whitelist verification
|
||||
if !labels.AreLabelsInWhiteList(podNodeSelectorLabels, whitelist) {
|
||||
return errors.NewForbidden(resource, name, fmt.Errorf("pod node label selector labels conflict with its namespace whitelist"))
|
||||
}
|
||||
|
||||
// Updated pod node selector = namespace node selector + current pod node selector
|
||||
pod.Spec.NodeSelector = map[string]string(podNodeSelectorLabels)
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewPodNodeSelector(client clientset.Interface, clusterNodeSelectors map[string]string) *podNodeSelector {
|
||||
return &podNodeSelector{
|
||||
Handler: admission.NewHandler(admission.Create),
|
||||
client: client,
|
||||
clusterNodeSelectors: clusterNodeSelectors,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *podNodeSelector) SetInformerFactory(f informers.SharedInformerFactory) {
|
||||
p.namespaceInformer = f.Namespaces().Informer()
|
||||
p.SetReadyFunc(p.namespaceInformer.HasSynced)
|
||||
}
|
||||
|
||||
func (p *podNodeSelector) Validate() error {
|
||||
if p.namespaceInformer == nil {
|
||||
return fmt.Errorf("missing namespaceInformer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *podNodeSelector) defaultGetNamespace(name string) (*api.Namespace, error) {
|
||||
namespace, err := p.client.Core().Namespaces().Get(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("namespace %s does not exist", name)
|
||||
}
|
||||
return namespace, nil
|
||||
}
|
||||
|
||||
func (p *podNodeSelector) getNodeSelectorMap(namespace *api.Namespace) (labels.Set, error) {
|
||||
selector := labels.Set{}
|
||||
labelsMap := labels.Set{}
|
||||
var err error
|
||||
found := false
|
||||
if len(namespace.ObjectMeta.Annotations) > 0 {
|
||||
for _, annotation := range NamespaceNodeSelectors {
|
||||
if ns, ok := namespace.ObjectMeta.Annotations[annotation]; ok {
|
||||
labelsMap, err = labels.ConvertSelectorToLabelsMap(ns)
|
||||
if err != nil {
|
||||
return labels.Set{}, err
|
||||
}
|
||||
|
||||
if labels.Conflicts(selector, labelsMap) {
|
||||
nsName := namespace.ObjectMeta.Name
|
||||
return labels.Set{}, fmt.Errorf("%s annotations' node label selectors conflict", nsName)
|
||||
}
|
||||
selector = labels.Merge(selector, labelsMap)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
selector, err = labels.ConvertSelectorToLabelsMap(p.clusterNodeSelectors["clusterDefaultNodeSelector"])
|
||||
if err != nil {
|
||||
return labels.Set{}, err
|
||||
}
|
||||
}
|
||||
return selector, nil
|
||||
}
|
||||
188
plugin/pkg/admission/podnodeselector/admission_test.go
Normal file
188
plugin/pkg/admission/podnodeselector/admission_test.go
Normal file
@@ -0,0 +1,188 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package podnodeselector
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/kubernetes/pkg/admission"
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
|
||||
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
|
||||
"k8s.io/kubernetes/pkg/controller/informers"
|
||||
"k8s.io/kubernetes/pkg/labels"
|
||||
"k8s.io/kubernetes/pkg/util/wait"
|
||||
)
|
||||
|
||||
// TestPodAdmission verifies various scenarios involving pod/namespace/global node label selectors
|
||||
func TestPodAdmission(t *testing.T) {
|
||||
namespace := &api.Namespace{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Name: "testNamespace",
|
||||
Namespace: "",
|
||||
},
|
||||
}
|
||||
|
||||
mockClient := &fake.Clientset{}
|
||||
handler, informerFactory, err := newHandlerForTest(mockClient)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error initializing handler: %v", err)
|
||||
}
|
||||
informerFactory.Start(wait.NeverStop)
|
||||
|
||||
pod := &api.Pod{
|
||||
ObjectMeta: api.ObjectMeta{Name: "testPod", Namespace: "testNamespace"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
defaultNodeSelector string
|
||||
namespaceNodeSelector string
|
||||
whitelist string
|
||||
podNodeSelector map[string]string
|
||||
mergedNodeSelector labels.Set
|
||||
ignoreTestNamespaceNodeSelector bool
|
||||
admit bool
|
||||
testName string
|
||||
}{
|
||||
{
|
||||
defaultNodeSelector: "",
|
||||
podNodeSelector: map[string]string{},
|
||||
mergedNodeSelector: labels.Set{},
|
||||
ignoreTestNamespaceNodeSelector: true,
|
||||
admit: true,
|
||||
testName: "No node selectors",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "infra = false",
|
||||
podNodeSelector: map[string]string{},
|
||||
mergedNodeSelector: labels.Set{"infra": "false"},
|
||||
ignoreTestNamespaceNodeSelector: true,
|
||||
admit: true,
|
||||
testName: "Default node selector and no conflicts",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "",
|
||||
namespaceNodeSelector: " infra = false ",
|
||||
podNodeSelector: map[string]string{},
|
||||
mergedNodeSelector: labels.Set{"infra": "false"},
|
||||
admit: true,
|
||||
testName: "TestNamespace node selector with whitespaces and no conflicts",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "infra = false",
|
||||
namespaceNodeSelector: "infra=true",
|
||||
podNodeSelector: map[string]string{},
|
||||
mergedNodeSelector: labels.Set{"infra": "true"},
|
||||
admit: true,
|
||||
testName: "Default and namespace node selector, no conflicts",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "infra = false",
|
||||
namespaceNodeSelector: "",
|
||||
podNodeSelector: map[string]string{},
|
||||
mergedNodeSelector: labels.Set{},
|
||||
admit: true,
|
||||
testName: "Empty namespace node selector and no conflicts",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "infra = false",
|
||||
namespaceNodeSelector: "infra=true",
|
||||
podNodeSelector: map[string]string{"env": "test"},
|
||||
mergedNodeSelector: labels.Set{"infra": "true", "env": "test"},
|
||||
admit: true,
|
||||
testName: "TestNamespace and pod node selector, no conflicts",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "env = test",
|
||||
namespaceNodeSelector: "infra=true",
|
||||
podNodeSelector: map[string]string{"infra": "false"},
|
||||
admit: false,
|
||||
testName: "Conflicting pod and namespace node selector, one label",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "env=dev",
|
||||
namespaceNodeSelector: "infra=false, env = test",
|
||||
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
|
||||
admit: false,
|
||||
testName: "Conflicting pod and namespace node selector, multiple labels",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "env=dev",
|
||||
namespaceNodeSelector: "infra=false, env = dev",
|
||||
whitelist: "env=dev, infra=false, color=blue",
|
||||
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
|
||||
mergedNodeSelector: labels.Set{"infra": "false", "env": "dev", "color": "blue"},
|
||||
admit: true,
|
||||
testName: "Merged pod node selectors satisfy the whitelist",
|
||||
},
|
||||
{
|
||||
defaultNodeSelector: "env=dev",
|
||||
namespaceNodeSelector: "infra=false, env = dev",
|
||||
whitelist: "env=dev, infra=true, color=blue",
|
||||
podNodeSelector: map[string]string{"env": "dev", "color": "blue"},
|
||||
admit: false,
|
||||
testName: "Merged pod node selectors conflict with the whitelist",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
if !test.ignoreTestNamespaceNodeSelector {
|
||||
namespace.ObjectMeta.Annotations = map[string]string{"scheduler.alpha.kubernetes.io/node-selector": test.namespaceNodeSelector}
|
||||
handler.namespaceInformer.GetStore().Update(namespace)
|
||||
}
|
||||
handler.clusterNodeSelectors = make(map[string]string)
|
||||
handler.clusterNodeSelectors["clusterDefaultNodeSelector"] = test.defaultNodeSelector
|
||||
handler.clusterNodeSelectors[namespace.Name] = test.whitelist
|
||||
pod.Spec = api.PodSpec{NodeSelector: test.podNodeSelector}
|
||||
|
||||
err := handler.Admit(admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Create, nil))
|
||||
if test.admit && err != nil {
|
||||
t.Errorf("Test: %s, expected no error but got: %s", test.testName, err)
|
||||
} else if !test.admit && err == nil {
|
||||
t.Errorf("Test: %s, expected an error", test.testName)
|
||||
}
|
||||
|
||||
if test.admit && !labels.Equals(test.mergedNodeSelector, labels.Set(pod.Spec.NodeSelector)) {
|
||||
t.Errorf("Test: %s, expected: %s but got: %s", test.testName, test.mergedNodeSelector, pod.Spec.NodeSelector)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandles(t *testing.T) {
|
||||
for op, shouldHandle := range map[admission.Operation]bool{
|
||||
admission.Create: true,
|
||||
admission.Update: false,
|
||||
admission.Connect: false,
|
||||
admission.Delete: false,
|
||||
} {
|
||||
nodeEnvionment := NewPodNodeSelector(nil, nil)
|
||||
if e, a := shouldHandle, nodeEnvionment.Handles(op); e != a {
|
||||
t.Errorf("%v: shouldHandle=%t, handles=%t", op, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newHandlerForTest returns the admission controller configured for testing.
|
||||
func newHandlerForTest(c clientset.Interface) (*podNodeSelector, informers.SharedInformerFactory, error) {
|
||||
f := informers.NewSharedInformerFactory(c, 5*time.Minute)
|
||||
handler := NewPodNodeSelector(c, nil)
|
||||
plugins := []admission.Interface{handler}
|
||||
pluginInitializer := admission.NewPluginInitializer(f, nil)
|
||||
pluginInitializer.Initialize(plugins)
|
||||
err := admission.Validate(plugins)
|
||||
return handler, f, err
|
||||
}
|
||||
Reference in New Issue
Block a user