dd9402f451
Kube-proxy must atleast wait for some duration for the node to defined so that it can fetch the node IP. This node IP is currently used in ipvs proxier for the node-port services.
333 lines
12 KiB
Go
333 lines
12 KiB
Go
// +build !windows
|
|
|
|
/*
|
|
Copyright 2014 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
// Package app does all of the work necessary to configure and run a
|
|
// Kubernetes app process.
|
|
package app
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
|
|
"k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
utilnet "k8s.io/apimachinery/pkg/util/net"
|
|
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
|
"k8s.io/client-go/tools/record"
|
|
"k8s.io/kubernetes/pkg/proxy"
|
|
proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
|
|
proxyconfig "k8s.io/kubernetes/pkg/proxy/config"
|
|
"k8s.io/kubernetes/pkg/proxy/healthcheck"
|
|
"k8s.io/kubernetes/pkg/proxy/iptables"
|
|
"k8s.io/kubernetes/pkg/proxy/ipvs"
|
|
"k8s.io/kubernetes/pkg/proxy/metrics"
|
|
"k8s.io/kubernetes/pkg/proxy/userspace"
|
|
"k8s.io/kubernetes/pkg/util/configz"
|
|
utildbus "k8s.io/kubernetes/pkg/util/dbus"
|
|
utilipset "k8s.io/kubernetes/pkg/util/ipset"
|
|
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
|
|
utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
|
|
utilnode "k8s.io/kubernetes/pkg/util/node"
|
|
utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
|
|
"k8s.io/utils/exec"
|
|
|
|
"k8s.io/klog"
|
|
)
|
|
|
|
// NewProxyServer returns a new ProxyServer.
|
|
func NewProxyServer(o *Options) (*ProxyServer, error) {
|
|
return newProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master)
|
|
}
|
|
|
|
func newProxyServer(
|
|
config *proxyconfigapi.KubeProxyConfiguration,
|
|
cleanupAndExit bool,
|
|
cleanupIPVS bool,
|
|
scheme *runtime.Scheme,
|
|
master string) (*ProxyServer, error) {
|
|
|
|
if config == nil {
|
|
return nil, errors.New("config is required")
|
|
}
|
|
|
|
if c, err := configz.New(proxyconfigapi.GroupName); err == nil {
|
|
c.Set(config)
|
|
} else {
|
|
return nil, fmt.Errorf("unable to register configz: %s", err)
|
|
}
|
|
|
|
protocol := utiliptables.ProtocolIpv4
|
|
if net.ParseIP(config.BindAddress).To4() == nil {
|
|
klog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress)
|
|
protocol = utiliptables.ProtocolIpv6
|
|
}
|
|
|
|
var iptInterface utiliptables.Interface
|
|
var ipvsInterface utilipvs.Interface
|
|
var kernelHandler ipvs.KernelHandler
|
|
var ipsetInterface utilipset.Interface
|
|
var dbus utildbus.Interface
|
|
|
|
// Create a iptables utils.
|
|
execer := exec.New()
|
|
|
|
dbus = utildbus.New()
|
|
iptInterface = utiliptables.New(execer, dbus, protocol)
|
|
kernelHandler = ipvs.NewLinuxKernelHandler()
|
|
ipsetInterface = utilipset.New(execer)
|
|
canUseIPVS, _ := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface)
|
|
if canUseIPVS {
|
|
ipvsInterface = utilipvs.New(execer)
|
|
}
|
|
|
|
// We omit creation of pretty much everything if we run in cleanup mode
|
|
if cleanupAndExit {
|
|
return &ProxyServer{
|
|
execer: execer,
|
|
IptInterface: iptInterface,
|
|
IpvsInterface: ipvsInterface,
|
|
IpsetInterface: ipsetInterface,
|
|
CleanupAndExit: cleanupAndExit,
|
|
}, nil
|
|
}
|
|
|
|
client, eventClient, err := createClients(config.ClientConnection, master)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Create event recorder
|
|
hostname, err := utilnode.GetHostname(config.HostnameOverride)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
eventBroadcaster := record.NewBroadcaster()
|
|
recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: "kube-proxy", Host: hostname})
|
|
|
|
nodeRef := &v1.ObjectReference{
|
|
Kind: "Node",
|
|
Name: hostname,
|
|
UID: types.UID(hostname),
|
|
Namespace: "",
|
|
}
|
|
|
|
var healthzServer *healthcheck.HealthzServer
|
|
var healthzUpdater healthcheck.HealthzUpdater
|
|
if len(config.HealthzBindAddress) > 0 {
|
|
healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef)
|
|
healthzUpdater = healthzServer
|
|
}
|
|
|
|
var proxier proxy.ProxyProvider
|
|
var serviceEventHandler proxyconfig.ServiceHandler
|
|
var endpointsEventHandler proxyconfig.EndpointsHandler
|
|
|
|
proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{})
|
|
nodeIP := net.ParseIP(config.BindAddress)
|
|
if nodeIP.IsUnspecified() {
|
|
nodeIP = utilnode.GetNodeIP(client, hostname)
|
|
if nodeIP == nil {
|
|
return nil, fmt.Errorf("unable to get node IP for hostname %s", hostname)
|
|
}
|
|
}
|
|
if proxyMode == proxyModeIPTables {
|
|
klog.V(0).Info("Using iptables Proxier.")
|
|
if config.IPTables.MasqueradeBit == nil {
|
|
// MasqueradeBit must be specified or defaulted.
|
|
return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config")
|
|
}
|
|
|
|
// TODO this has side effects that should only happen when Run() is invoked.
|
|
proxierIPTables, err := iptables.NewProxier(
|
|
iptInterface,
|
|
utilsysctl.New(),
|
|
execer,
|
|
config.IPTables.SyncPeriod.Duration,
|
|
config.IPTables.MinSyncPeriod.Duration,
|
|
config.IPTables.MasqueradeAll,
|
|
int(*config.IPTables.MasqueradeBit),
|
|
config.ClusterCIDR,
|
|
hostname,
|
|
nodeIP,
|
|
recorder,
|
|
healthzUpdater,
|
|
config.NodePortAddresses,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to create proxier: %v", err)
|
|
}
|
|
metrics.RegisterMetrics()
|
|
proxier = proxierIPTables
|
|
serviceEventHandler = proxierIPTables
|
|
endpointsEventHandler = proxierIPTables
|
|
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
|
|
klog.V(0).Info("Tearing down inactive rules.")
|
|
// TODO this has side effects that should only happen when Run() is invoked.
|
|
userspace.CleanupLeftovers(iptInterface)
|
|
// IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode.
|
|
// Besides, ipvs proxier will create some ipvs rules as well. Because there is no way to tell if a given
|
|
// ipvs rule is created by IPVS proxier or not. Users should explicitly specify `--clean-ipvs=true` to flush
|
|
// all ipvs rules when kube-proxy start up. Users do this operation should be with caution.
|
|
if canUseIPVS {
|
|
ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
|
|
}
|
|
} else if proxyMode == proxyModeIPVS {
|
|
klog.V(0).Info("Using ipvs Proxier.")
|
|
proxierIPVS, err := ipvs.NewProxier(
|
|
iptInterface,
|
|
ipvsInterface,
|
|
ipsetInterface,
|
|
utilsysctl.New(),
|
|
execer,
|
|
config.IPVS.SyncPeriod.Duration,
|
|
config.IPVS.MinSyncPeriod.Duration,
|
|
config.IPVS.ExcludeCIDRs,
|
|
config.IPTables.MasqueradeAll,
|
|
int(*config.IPTables.MasqueradeBit),
|
|
config.ClusterCIDR,
|
|
hostname,
|
|
nodeIP,
|
|
recorder,
|
|
healthzServer,
|
|
config.IPVS.Scheduler,
|
|
config.NodePortAddresses,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to create proxier: %v", err)
|
|
}
|
|
metrics.RegisterMetrics()
|
|
proxier = proxierIPVS
|
|
serviceEventHandler = proxierIPVS
|
|
endpointsEventHandler = proxierIPVS
|
|
klog.V(0).Info("Tearing down inactive rules.")
|
|
// TODO this has side effects that should only happen when Run() is invoked.
|
|
userspace.CleanupLeftovers(iptInterface)
|
|
iptables.CleanupLeftovers(iptInterface)
|
|
} else {
|
|
klog.V(0).Info("Using userspace Proxier.")
|
|
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
|
|
// our config.EndpointsConfigHandler.
|
|
loadBalancer := userspace.NewLoadBalancerRR()
|
|
// set EndpointsConfigHandler to our loadBalancer
|
|
endpointsEventHandler = loadBalancer
|
|
|
|
// TODO this has side effects that should only happen when Run() is invoked.
|
|
proxierUserspace, err := userspace.NewProxier(
|
|
loadBalancer,
|
|
net.ParseIP(config.BindAddress),
|
|
iptInterface,
|
|
execer,
|
|
*utilnet.ParsePortRangeOrDie(config.PortRange),
|
|
config.IPTables.SyncPeriod.Duration,
|
|
config.IPTables.MinSyncPeriod.Duration,
|
|
config.UDPIdleTimeout.Duration,
|
|
config.NodePortAddresses,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to create proxier: %v", err)
|
|
}
|
|
serviceEventHandler = proxierUserspace
|
|
proxier = proxierUserspace
|
|
|
|
// Remove artifacts from the iptables and ipvs Proxier, if not on Windows.
|
|
klog.V(0).Info("Tearing down inactive rules.")
|
|
// TODO this has side effects that should only happen when Run() is invoked.
|
|
iptables.CleanupLeftovers(iptInterface)
|
|
// IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode.
|
|
// Besides, ipvs proxier will create some ipvs rules as well. Because there is no way to tell if a given
|
|
// ipvs rule is created by IPVS proxier or not. Users should explicitly specify `--clean-ipvs=true` to flush
|
|
// all ipvs rules when kube-proxy start up. Users do this operation should be with caution.
|
|
if canUseIPVS {
|
|
ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
|
|
}
|
|
}
|
|
|
|
iptInterface.AddReloadFunc(proxier.Sync)
|
|
|
|
return &ProxyServer{
|
|
Client: client,
|
|
EventClient: eventClient,
|
|
IptInterface: iptInterface,
|
|
IpvsInterface: ipvsInterface,
|
|
IpsetInterface: ipsetInterface,
|
|
execer: execer,
|
|
Proxier: proxier,
|
|
Broadcaster: eventBroadcaster,
|
|
Recorder: recorder,
|
|
ConntrackConfiguration: config.Conntrack,
|
|
Conntracker: &realConntracker{},
|
|
ProxyMode: proxyMode,
|
|
NodeRef: nodeRef,
|
|
MetricsBindAddress: config.MetricsBindAddress,
|
|
EnableProfiling: config.EnableProfiling,
|
|
OOMScoreAdj: config.OOMScoreAdj,
|
|
ResourceContainer: config.ResourceContainer,
|
|
ConfigSyncPeriod: config.ConfigSyncPeriod.Duration,
|
|
ServiceEventHandler: serviceEventHandler,
|
|
EndpointsEventHandler: endpointsEventHandler,
|
|
HealthzServer: healthzServer,
|
|
}, nil
|
|
}
|
|
|
|
func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
|
|
switch proxyMode {
|
|
case proxyModeUserspace:
|
|
return proxyModeUserspace
|
|
case proxyModeIPTables:
|
|
return tryIPTablesProxy(iptver, kcompat)
|
|
case proxyModeIPVS:
|
|
return tryIPVSProxy(iptver, khandle, ipsetver, kcompat)
|
|
}
|
|
klog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode)
|
|
return tryIPTablesProxy(iptver, kcompat)
|
|
}
|
|
|
|
func tryIPVSProxy(iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
|
|
// guaranteed false on error, error only necessary for debugging
|
|
// IPVS Proxier relies on ip_vs_* kernel modules and ipset
|
|
useIPVSProxy, err := ipvs.CanUseIPVSProxier(khandle, ipsetver)
|
|
if err != nil {
|
|
// Try to fallback to iptables before falling back to userspace
|
|
utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, error: %v", err))
|
|
}
|
|
if useIPVSProxy {
|
|
return proxyModeIPVS
|
|
}
|
|
|
|
// Try to fallback to iptables before falling back to userspace
|
|
klog.V(1).Infof("Can't use ipvs proxier, trying iptables proxier")
|
|
return tryIPTablesProxy(iptver, kcompat)
|
|
}
|
|
|
|
func tryIPTablesProxy(iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string {
|
|
// guaranteed false on error, error only necessary for debugging
|
|
useIPTablesProxy, err := iptables.CanUseIPTablesProxier(iptver, kcompat)
|
|
if err != nil {
|
|
utilruntime.HandleError(fmt.Errorf("can't determine whether to use iptables proxy, using userspace proxier: %v", err))
|
|
return proxyModeUserspace
|
|
}
|
|
if useIPTablesProxy {
|
|
return proxyModeIPTables
|
|
}
|
|
// Fallback.
|
|
klog.V(1).Infof("Can't use iptables proxy, using userspace proxier")
|
|
return proxyModeUserspace
|
|
}
|