github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Distinguish iptables-based and nftables-based backends, do startup cleanup

Browse Source 
When switching from iptables or ipvs to nftables, clean up old
iptables/ipvs rules. When switching the other way, clean up old
nftables rules.
This commit is contained in:
Dan Winship
2023-10-23 09:38:24 -04:00
parent abb1a458a9
commit 93860a5217
4 changed files with 105 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/kube-proxy/app/server.go
View File

@@ -375,9 +375,12 @@ func (o *Options) Run() error {
return o.writeConfigFile()
}
err := platformCleanup(o.config.Mode, o.CleanupAndExit)
if o.CleanupAndExit {
return cleanupAndExit()
return err
}
// We ignore err otherwise; the cleanup is best-effort, and the backends will have
// logged messages if they failed in interesting ways.
proxyServer, err := newProxyServer(o.config, o.master, o.InitAndExit)
if err != nil {

145
cmd/kube-proxy/app/server_others.go
View File

@@ -40,9 +40,11 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
toolswatch "k8s.io/client-go/tools/watch"
utilsysctl "k8s.io/component-helpers/node/util/sysctl"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/proxy/iptables"
@@ -101,60 +103,73 @@ func (s *ProxyServer) platformSetup() error {
return nil
}
// isIPTablesBased checks whether mode is based on iptables rather than nftables
func isIPTablesBased(mode proxyconfigapi.ProxyMode) bool {
return mode == proxyconfigapi.ProxyModeIPTables || mode == proxyconfigapi.ProxyModeIPVS
}
// getIPTables returns an array of [IPv4, IPv6] utiliptables.Interfaces. If primaryFamily
// is not v1.IPFamilyUnknown then it will also separately return the interface for just
// that family.
func getIPTables(primaryFamily v1.IPFamily) ([2]utiliptables.Interface, utiliptables.Interface) {
execer := exec.New()
// Create iptables handlers for both families. Always ordered as IPv4, IPv6
ipt := [2]utiliptables.Interface{
utiliptables.New(execer, utiliptables.ProtocolIPv4),
utiliptables.New(execer, utiliptables.ProtocolIPv6),
}
var iptInterface utiliptables.Interface
if primaryFamily == v1.IPv4Protocol {
iptInterface = ipt[0]
} else if primaryFamily == v1.IPv6Protocol {
iptInterface = ipt[1]
}
return ipt, iptInterface
}
// platformCheckSupported is called immediately before creating the Proxier, to check
// what IP families are supported (and whether the configuration is usable at all).
func (s *ProxyServer) platformCheckSupported() (ipv4Supported, ipv6Supported, dualStackSupported bool, err error) {
execer := exec.New()
ipt := utiliptables.New(execer, utiliptables.ProtocolIPv4)
ipv4Supported = ipt.Present()
ipt = utiliptables.New(execer, utiliptables.ProtocolIPv6)
ipv6Supported = ipt.Present()
if isIPTablesBased(s.Config.Mode) {
ipt, _ := getIPTables(v1.IPFamilyUnknown)
ipv4Supported = ipt[0].Present()
ipv6Supported = ipt[1].Present()
if !ipv4Supported && !ipv6Supported {
err = fmt.Errorf("iptables is not available on this host")
} else if !ipv4Supported {
klog.InfoS("No iptables support for family", "ipFamily", v1.IPv4Protocol)
} else if !ipv6Supported {
klog.InfoS("No iptables support for family", "ipFamily", v1.IPv6Protocol)
}
} else {
// Assume support for both families.
// FIXME: figure out how to check for kernel IPv6 support using nft
ipv4Supported, ipv6Supported = true, true
}
// The Linux proxies can always support dual-stack if they can support both IPv4
// and IPv6.
dualStackSupported = ipv4Supported && ipv6Supported
if !ipv4Supported && !ipv6Supported {
err = fmt.Errorf("iptables is not available on this host")
} else if !ipv4Supported {
klog.InfoS("No iptables support for family", "ipFamily", v1.IPv4Protocol)
} else if !ipv6Supported {
klog.InfoS("No iptables support for family", "ipFamily", v1.IPv6Protocol)
}
return
}
// createProxier creates the proxy.Provider
func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguration, dualStack, initOnly bool) (proxy.Provider, error) {
var proxier proxy.Provider
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
var localDetector proxyutiliptables.LocalTrafficDetector
var err error
primaryProtocol := utiliptables.ProtocolIPv4
if s.PrimaryIPFamily == v1.IPv6Protocol {
primaryProtocol = utiliptables.ProtocolIPv6
}
execer := exec.New()
iptInterface := utiliptables.New(execer, primaryProtocol)
var ipt [2]utiliptables.Interface
// Create iptables handlers for both families, one is already created
// Always ordered as IPv4, IPv6
if primaryProtocol == utiliptables.ProtocolIPv4 {
ipt[0] = iptInterface
ipt[1] = utiliptables.New(execer, utiliptables.ProtocolIPv6)
} else {
ipt[0] = utiliptables.New(execer, utiliptables.ProtocolIPv4)
ipt[1] = iptInterface
}
if config.Mode == proxyconfigapi.ProxyModeIPTables {
klog.InfoS("Using iptables Proxier")
if dualStack {
// Always ordered to match []ipt
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
ipt, _ := getIPTables(s.PrimaryIPFamily)
localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -164,7 +179,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
proxier, err = iptables.NewDualStackProxier(
ipt,
utilsysctl.New(),
execer,
exec.New(),
config.IPTables.SyncPeriod.Duration,
config.IPTables.MinSyncPeriod.Duration,
config.IPTables.MasqueradeAll,
@@ -180,7 +195,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
)
} else {
// Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support).
var localDetector proxyutiliptables.LocalTrafficDetector
_, iptInterface := getIPTables(s.PrimaryIPFamily)
localDetector, err = getLocalDetector(s.PrimaryIPFamily, config.DetectLocalMode, config, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -191,7 +206,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
s.PrimaryIPFamily,
iptInterface,
utilsysctl.New(),
execer,
exec.New(),
config.IPTables.SyncPeriod.Duration,
config.IPTables.MinSyncPeriod.Duration,
config.IPTables.MasqueradeAll,
@@ -211,6 +226,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
} else if config.Mode == proxyconfigapi.ProxyModeIPVS {
execer := exec.New()
ipsetInterface := utilipset.New(execer)
ipvsInterface := utilipvs.New()
if err := ipvs.CanUseIPVSProxier(ipvsInterface, ipsetInterface, config.IPVS.Scheduler); err != nil {
@@ -219,8 +235,9 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
klog.InfoS("Using ipvs Proxier")
if dualStack {
ipt, _ := getIPTables(s.PrimaryIPFamily)
// Always ordered to match []ipt
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -251,7 +268,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
initOnly,
)
} else {
var localDetector proxyutiliptables.LocalTrafficDetector
_, iptInterface := getIPTables(s.PrimaryIPFamily)
localDetector, err = getLocalDetector(s.PrimaryIPFamily, config.DetectLocalMode, config, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -290,8 +307,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
klog.InfoS("Using nftables Proxier")
if dualStack {
// Always ordered to match []ipt
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
localDetectors, err = getDualStackLocalDetectorTuple(config.DetectLocalMode, config, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -313,8 +328,7 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
initOnly,
)
} else {
// Create a single-stack proxier if and only if the node does not support dual-stack (i.e, no iptables support).
var localDetector proxyutiliptables.LocalTrafficDetector
// Create a single-stack proxier if and only if the node does not support dual-stack
localDetector, err = getLocalDetector(s.PrimaryIPFamily, config.DetectLocalMode, config, s.podCIDRs)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -531,28 +545,35 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy
return localDetectors, nil
}
// cleanupAndExit remove iptables rules and ipset/ipvs rules
func cleanupAndExit() error {
execer := exec.New()
// cleanup IPv6 and IPv4 iptables rules, regardless of current configuration
ipts := []utiliptables.Interface{
utiliptables.New(execer, utiliptables.ProtocolIPv4),
utiliptables.New(execer, utiliptables.ProtocolIPv6),
}
ipsetInterface := utilipset.New(execer)
ipvsInterface := utilipvs.New()
// platformCleanup removes stale kube-proxy rules that can be safely removed. If
// cleanupAndExit is true, it will attempt to remove rules from all known kube-proxy
// modes. If it is false, it will only remove rules that are definitely not in use by the
// currently-configured mode.
func platformCleanup(mode proxyconfigapi.ProxyMode, cleanupAndExit bool) error {
var encounteredError bool
for _, ipt := range ipts {
encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
encounteredError = ipvs.CleanupLeftovers(ipvsInterface, ipt, ipsetInterface) || encounteredError
encounteredError = nftables.CleanupLeftovers(ipt) || encounteredError
// Clean up iptables and ipvs rules if switching to nftables, or if cleanupAndExit
if !isIPTablesBased(mode) || cleanupAndExit {
ipts, _ := getIPTables(v1.IPFamilyUnknown)
execer := exec.New()
ipsetInterface := utilipset.New(execer)
ipvsInterface := utilipvs.New()
for _, ipt := range ipts {
encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
encounteredError = ipvs.CleanupLeftovers(ipvsInterface, ipt, ipsetInterface) || encounteredError
}
}
if utilfeature.DefaultFeatureGate.Enabled(features.NFTablesProxyMode) {
// Clean up nftables rules when switching to iptables or ipvs, or if cleanupAndExit
if isIPTablesBased(mode) || cleanupAndExit {
encounteredError = nftables.CleanupLeftovers() || encounteredError
}
}
if encounteredError {
return errors.New("encountered an error while tearing down rules")
}
return nil
}

9
cmd/kube-proxy/app/server_windows.go
View File

@@ -125,7 +125,10 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
return proxier, nil
}
// cleanupAndExit cleans up after a previous proxy run
func cleanupAndExit() error {
return errors.New("--cleanup-and-exit is not implemented on Windows")
// platformCleanup removes stale kube-proxy rules that can be safely removed.
func platformCleanup(mode proxyconfigapi.ProxyMode, cleanupAndExit bool) error {
if cleanupAndExit {
return errors.New("--cleanup-and-exit is not implemented on Windows")
}
return nil
}