github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: disable insecure serving in controller-manager

Browse Source 
Now the following flags have no effect and would be removed in v1.24:
* `--port`
* `--address`

The insecure port flags `--port` may only be set to 0 now.

Signed-off-by: Jian Zeng <zengjian.zj@bytedance.com>
This commit is contained in:
Jian Zeng 2021-05-03 00:01:49 +08:00
parent f79795d718
commit e481d99965
No known key found for this signature in database
GPG Key ID: 1040B69865E7D86C
7 changed files with 50 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cluster/gce/manifests/kube-controller-manager.manifest
View File

@ -46,7 +46,8 @@
"livenessProbe": {
"httpGet": {
"host": "127.0.0.1",
"port": 10252,
"port": 10257,
"scheme": "HTTPS",
"path": "/healthz"
},
"initialDelaySeconds": 15,

29
cmd/kube-controller-manager/app/controllermanager.go
View File

@ -30,6 +30,7 @@ import (
"time"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@ -37,7 +38,6 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
genericfeatures "k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/mux"
utilfeature "k8s.io/apiserver/pkg/util/feature"
@ -62,6 +62,7 @@ import (
"k8s.io/controller-manager/pkg/clientbuilder"
"k8s.io/controller-manager/pkg/informerfactory"
"k8s.io/klog/v2"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
@ -86,6 +87,18 @@ const (
ExternalLoops
)
// TODO: delete this check after insecure flags removed in v1.24
func checkNonZeroInsecurePort(fs *pflag.FlagSet) error {
val, err := fs.GetInt("port")
if err != nil {
return err
}
if val != 0 {
return fmt.Errorf("invalid port value %d: only zero is allowed", val)
}
return nil
}
// NewControllerManagerCommand creates a *cobra.Command object with default parameters
func NewControllerManagerCommand() *cobra.Command {
s, err := options.NewKubeControllerManagerOptions()
@ -114,6 +127,12 @@ controller, and serviceaccounts controller.`,
verflag.PrintAndExitIfRequested()
cliflag.PrintFlags(cmd.Flags())
err := checkNonZeroInsecurePort(cmd.Flags())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
@ -198,14 +217,6 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
return err
}
}
if c.InsecureServing != nil {
unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}}
handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn)
if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
return err
}
}
run := func(ctx context.Context) {
rootClientBuilder := clientbuilder.SimpleControllerClientBuilder{

51
cmd/kube-controller-manager/app/options/options.go
View File

@ -22,6 +22,7 @@ import (
"fmt"
"net"
"github.com/spf13/pflag"
v1 "k8s.io/api/core/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
@ -84,13 +85,11 @@ type KubeControllerManagerOptions struct {
SAController *SAControllerOptions
TTLAfterFinishedController *TTLAfterFinishedControllerOptions
SecureServing *apiserveroptions.SecureServingOptionsWithLoopback
// TODO: remove insecure serving mode
InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback
Authentication *apiserveroptions.DelegatingAuthenticationOptions
Authorization *apiserveroptions.DelegatingAuthorizationOptions
Metrics *metrics.Options
Logs *logs.Options
SecureServing *apiserveroptions.SecureServingOptionsWithLoopback
Authentication *apiserveroptions.DelegatingAuthenticationOptions
Authorization *apiserveroptions.DelegatingAuthorizationOptions
Metrics *metrics.Options
Logs *logs.Options
Master string
Kubeconfig string
@ -99,7 +98,7 @@ type KubeControllerManagerOptions struct {
// NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
componentConfig, err := NewDefaultComponentConfig(ports.InsecureKubeControllerManagerPort)
componentConfig, err := NewDefaultComponentConfig()
if err != nil {
return nil, err
}
@ -179,12 +178,7 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{
&componentConfig.TTLAfterFinishedController,
},
SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
BindAddress: net.ParseIP(componentConfig.Generic.Address),
BindPort: int(componentConfig.Generic.Port),
BindNetwork: "tcp",
}).WithLoopback(),
SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(),
Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(),
Metrics: metrics.NewOptions(),
@ -212,7 +206,7 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
}
// NewDefaultComponentConfig returns kube-controller manager configuration object.
func NewDefaultComponentConfig(insecurePort int32) (kubectrlmgrconfig.KubeControllerManagerConfiguration, error) {
func NewDefaultComponentConfig() (kubectrlmgrconfig.KubeControllerManagerConfiguration, error) {
versioned := kubectrlmgrconfigv1alpha1.KubeControllerManagerConfiguration{}
kubectrlmgrconfigscheme.Scheme.Default(&versioned)
@ -220,10 +214,23 @@ func NewDefaultComponentConfig(insecurePort int32) (kubectrlmgrconfig.KubeContro
if err := kubectrlmgrconfigscheme.Scheme.Convert(&versioned, &internal, nil); err != nil {
return internal, err
}
internal.Generic.Port = insecurePort
return internal, nil
}
// TODO: remove these insecure flags in v1.24
func addDummyInsecureFlags(fs *pflag.FlagSet) {
var (
bindAddr = net.IPv4(127, 0, 0, 1)
bindPort = 0
)
fs.IPVar(&bindAddr, "address", bindAddr,
"The IP address on which to serve the insecure --port (set to 0.0.0.0 for all IPv4 interfaces and :: for all IPv6 interfaces).")
fs.MarkDeprecated("address", "This flag has no effect now and will be removed in v1.24.")
fs.IntVar(&bindPort, "port", bindPort, "The port on which to serve unsecured, unauthenticated access. Set to 0 to disable.")
fs.MarkDeprecated("port", "This flag has no effect now and will be removed in v1.24.")
}
// Flags returns flags for a specific APIServer by section name
func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledByDefaultControllers []string) cliflag.NamedFlagSets {
fss := cliflag.NamedFlagSets{}
@ -232,7 +239,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
s.ServiceController.AddFlags(fss.FlagSet("service controller"))
s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
s.InsecureServing.AddUnqualifiedFlags(fss.FlagSet("insecure serving"))
addDummyInsecureFlags(fss.FlagSet("insecure serving"))
s.Authentication.AddFlags(fss.FlagSet("authentication"))
s.Authorization.AddFlags(fss.FlagSet("authorization"))
@ -350,9 +357,6 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e
if err := s.TTLAfterFinishedController.ApplyTo(&c.ComponentConfig.TTLAfterFinishedController); err != nil {
return err
}
if err := s.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil {
return err
}
if err := s.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
return err
}
@ -364,12 +368,6 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e
return err
}
}
// sync back to component config
// TODO: find more elegant way than syncing back the values.
c.ComponentConfig.Generic.Port = int32(s.InsecureServing.BindPort)
c.ComponentConfig.Generic.Address = s.InsecureServing.BindAddress.String()
return nil
}
@ -404,7 +402,6 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable
errs = append(errs, s.ServiceController.Validate()...)
errs = append(errs, s.TTLAfterFinishedController.Validate()...)
errs = append(errs, s.SecureServing.Validate()...)
errs = append(errs, s.InsecureServing.Validate()...)
errs = append(errs, s.Authentication.Validate()...)
errs = append(errs, s.Authorization.Validate()...)
errs = append(errs, s.Metrics.Validate()...)

13
cmd/kube-controller-manager/app/options/options_test.go
View File

@ -62,7 +62,6 @@ import (
)
var args = []string{
"--address=192.168.4.10",
"--allocate-node-cidrs=true",
"--attach-detach-reconcile-sync-period=30s",
"--cidr-allocator-type=CloudAllocator",
@ -136,7 +135,6 @@ var args = []string{
"--node-monitor-period=10s",
"--node-startup-grace-period=30s",
"--pod-eviction-timeout=2m",
"--port=10000",
"--profiling=false",
"--pv-recycler-increment-timeout-nfs=45",
"--pv-recycler-minimum-timeout-hostpath=45",
@ -171,8 +169,7 @@ func TestAddFlags(t *testing.T) {
expected := &KubeControllerManagerOptions{
Generic: &cmoptions.GenericControllerManagerConfigurationOptions{
GenericControllerManagerConfiguration: &cmconfig.GenericControllerManagerConfiguration{
Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
Address: "0.0.0.0", // Note: This field should have no effect in CM now, and "0.0.0.0" is the default value.
MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour},
ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
ContentType: "application/json",
@ -405,11 +402,6 @@ func TestAddFlags(t *testing.T) {
},
HTTP2MaxStreamsPerConnection: 47,
}).WithLoopback(),
InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
BindAddress: net.ParseIP("192.168.4.10"),
BindPort: int(10000),
BindNetwork: "tcp",
}).WithLoopback(),
Authentication: &apiserveroptions.DelegatingAuthenticationOptions{
CacheTTL: 10 * time.Second,
ClientTimeout: 10 * time.Second,
@ -462,8 +454,7 @@ func TestApplyTo(t *testing.T) {
expected := &kubecontrollerconfig.Config{
ComponentConfig: kubectrlmgrconfig.KubeControllerManagerConfiguration{
Generic: cmconfig.GenericControllerManagerConfiguration{
Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
Address: "0.0.0.0", // Note: This field should have no effect in CM now, and "0.0.0.0" is the default value.
MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour},
ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
ContentType: "application/json",

9
cmd/kube-controller-manager/app/testing/testserver.go
View File

@ -101,15 +101,6 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
t.Logf("kube-controller-manager will listen securely on port %d...", s.SecureServing.BindPort)
}
if s.InsecureServing.BindPort != 0 {
s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort()
if err != nil {
return result, fmt.Errorf("failed to create listener: %v", err)
}
t.Logf("kube-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
}
config, err := s.Config(all, disabled)
if err != nil {
return result, fmt.Errorf("failed to create config from options: %v", err)

4
pkg/cluster/ports/ports.go
View File

@ -25,10 +25,6 @@ const (
// KubeletPort is the default port for the kubelet server on each host machine.
// May be overridden by a flag at startup.
KubeletPort = 10250
// InsecureKubeControllerManagerPort is the default port for the controller manager status server.
// May be overridden by a flag at startup.
// Deprecated: use the secure KubeControllerManagerPort instead.
InsecureKubeControllerManagerPort = 10252
// KubeletReadOnlyPort exposes basic read-only services from the kubelet.
// May be overridden by a flag at startup.
// This is necessary for heapster to collect monitoring stats from the kubelet

3
pkg/registry/core/rest/storage_core.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package rest
import (
"crypto/tls"
"fmt"
"net"
"net/http"
@ -343,7 +344,7 @@ func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.
// this is fragile, which assumes that the default port is being used
// TODO: switch to secure port until these components remove the ability to serve insecurely.
serversToValidate := map[string]*componentstatus.Server{
"controller-manager": {Addr: "127.0.0.1", Port: ports.InsecureKubeControllerManagerPort, Path: "/healthz"},
"controller-manager": {EnableHTTPS: true, TLSConfig: &tls.Config{InsecureSkipVerify: true}, Addr: "127.0.0.1", Port: ports.KubeControllerManagerPort, Path: "/healthz"},
"scheduler": {Addr: "127.0.0.1", Port: kubeschedulerconfig.DefaultInsecureSchedulerPort, Path: "/healthz"},
}