github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Double-check the proxy configuration vs the available IP families

Browse Source
This commit is contained in:
Dan Winship
2023-06-30 12:25:11 -04:00
parent 1f2bf32e95
commit a966d18608
3 changed files with 401 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
cmd/kube-proxy/app/server.go
View File

@@ -41,6 +41,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/healthz"
@@ -583,6 +584,14 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin
klog.InfoS("kube-proxy running in single-stack mode", "ipFamily", s.PrimaryIPFamily)
}
err, fatal := checkIPConfig(s, dualStackSupported)
if err != nil {
if fatal {
return nil, fmt.Errorf("kube-proxy configuration is incorrect: %v", err)
}
klog.ErrorS(err, "Kube-proxy configuration may be incomplete or incorrect")
}
s.Proxier, err = s.createProxier(config, dualStackSupported)
if err != nil {
return nil, err
@@ -591,6 +600,99 @@ func newProxyServer(config *kubeproxyconfig.KubeProxyConfiguration, master strin
return s, nil
}
// checkIPConfig confirms that s has proper configuration for its primary IP family.
func checkIPConfig(s *ProxyServer, dualStackSupported bool) (error, bool) {
var errors []error
var badFamily netutils.IPFamily
if s.PrimaryIPFamily == v1.IPv4Protocol {
badFamily = netutils.IPv6
} else {
badFamily = netutils.IPv4
}
var clusterType string
if dualStackSupported {
clusterType = fmt.Sprintf("%s-primary", s.PrimaryIPFamily)
} else {
clusterType = fmt.Sprintf("%s-only", s.PrimaryIPFamily)
}
// Historically, we did not check most of the config options, so we cannot
// retroactively make IP family mismatches in those options be fatal. When we add
// new options to check here, we should make problems with those options be fatal.
fatal := false
if s.Config.ClusterCIDR != "" {
clusterCIDRs := strings.Split(s.Config.ClusterCIDR, ",")
if badCIDRs(clusterCIDRs, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but clusterCIDRs contains only IPv%s addresses", clusterType, badFamily))
if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeClusterCIDR {
// This has always been a fatal error
fatal = true
}
}
}
if badCIDRs(s.Config.NodePortAddresses, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but nodePortAddresses contains only IPv%s addresses", clusterType, badFamily))
}
if badCIDRs(s.podCIDRs, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but node.spec.podCIDRs contains only IPv%s addresses", clusterType, badFamily))
if s.Config.DetectLocalMode == kubeproxyconfig.LocalModeNodeCIDR {
// This has always been a fatal error
fatal = true
}
}
if netutils.IPFamilyOfString(s.Config.Winkernel.SourceVip) == badFamily {
errors = append(errors, fmt.Errorf("cluster is %s but winkernel.sourceVip is IPv%s", clusterType, badFamily))
}
// In some cases, wrong-IP-family is only a problem when the secondary IP family
// isn't present at all.
if !dualStackSupported {
if badCIDRs(s.Config.IPVS.ExcludeCIDRs, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but ipvs.excludeCIDRs contains only IPv%s addresses", clusterType, badFamily))
}
if badBindAddress(s.Config.HealthzBindAddress, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but healthzBindAddress is IPv%s", clusterType, badFamily))
}
if badBindAddress(s.Config.MetricsBindAddress, badFamily) {
errors = append(errors, fmt.Errorf("cluster is %s but metricsBindAddress is IPv%s", clusterType, badFamily))
}
}
return utilerrors.NewAggregate(errors), fatal
}
// badCIDRs returns true if cidrs is a non-empty list of CIDRs, all of wrongFamily.
func badCIDRs(cidrs []string, wrongFamily netutils.IPFamily) bool {
if len(cidrs) == 0 {
return false
}
for _, cidr := range cidrs {
if netutils.IPFamilyOfCIDRString(cidr) != wrongFamily {
return false
}
}
return true
}
// badBindAddress returns true if bindAddress is an "IP:port" string where IP is a
// non-zero IP of wrongFamily.
func badBindAddress(bindAddress string, wrongFamily netutils.IPFamily) bool {
if host, _, _ := net.SplitHostPort(bindAddress); host != "" {
ip := netutils.ParseIPSloppy(host)
if ip != nil && netutils.IPFamilyOf(ip) == wrongFamily && !ip.IsUnspecified() {
return true
}
}
return false
}
// createClient creates a kube client from the given config and masterOverride.
// TODO remove masterOverride when CLI flags are removed.
func createClient(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, error) {

11
cmd/kube-proxy/app/server_others.go
View File

@@ -50,7 +50,6 @@ import (
utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics"
proxyutil "k8s.io/kubernetes/pkg/proxy/util"
proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
"k8s.io/utils/exec"
@@ -149,16 +148,6 @@ func (s *ProxyServer) createProxier(config *proxyconfigapi.KubeProxyConfiguratio
ipt[1] = iptInterface
}
if !dualStack {
// Validate NodePortAddresses is single-stack
npaByFamily := proxyutil.MapCIDRsByIPFamily(config.NodePortAddresses)
secondaryFamily := proxyutil.OtherIPFamily(s.PrimaryIPFamily)
badAddrs := npaByFamily[secondaryFamily]
if len(badAddrs) > 0 {
klog.InfoS("Ignoring --nodeport-addresses of the wrong family", "ipFamily", secondaryFamily, "addresses", badAddrs)
}
}
if config.Mode == proxyconfigapi.ProxyModeIPTables {
klog.InfoS("Using iptables Proxier")

299
cmd/kube-proxy/app/server_test.go
View File

@@ -632,3 +632,302 @@ func Test_detectNodeIPs(t *testing.T) {
})
}
}
func Test_checkIPConfig(t *testing.T) {
cases := []struct {
name string
proxy *ProxyServer
ssErr bool
dsErr bool
fatal bool
}{
{
name: "empty config",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok single-stack clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "10.0.0.0/8",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok dual-stack clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "10.0.0.0/8,fd01:2345::/64",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok reversed dual-stack clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "fd01:2345::/64,10.0.0.0/8",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong-family clusterCIDR",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "fd01:2345::/64",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: true,
dsErr: true,
fatal: false,
},
{
name: "wrong-family clusterCIDR when using ClusterCIDR LocalDetector",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
ClusterCIDR: "fd01:2345::/64",
DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: true,
dsErr: true,
fatal: true,
},
{
name: "ok single-stack nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"10.0.0.0/8", "192.168.0.0/24"},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok dual-stack nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"10.0.0.0/8", "fd01:2345::/64", "fd01:abcd::/64"},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok reversed dual-stack nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"fd01:2345::/64", "fd01:abcd::/64", "10.0.0.0/8"},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong-family nodePortAddresses",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
NodePortAddresses: []string{"10.0.0.0/8"},
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: true,
dsErr: true,
fatal: false,
},
{
name: "ok single-stack node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"10.0.0.0/8"},
},
ssErr: false,
dsErr: false,
},
{
name: "ok dual-stack node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"10.0.0.0/8", "fd01:2345::/64"},
},
ssErr: false,
dsErr: false,
},
{
name: "ok reversed dual-stack node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"fd01:2345::/64", "10.0.0.0/8"},
},
ssErr: false,
dsErr: false,
},
{
name: "wrong-family node.spec.podCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
},
PrimaryIPFamily: v1.IPv4Protocol,
podCIDRs: []string{"fd01:2345::/64"},
},
ssErr: true,
dsErr: true,
fatal: true,
},
{
name: "ok winkernel.sourceVip",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{
SourceVip: "10.0.0.1",
},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong family winkernel.sourceVip",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{
SourceVip: "fd01:2345::1",
},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: true,
dsErr: true,
fatal: false,
},
{
name: "ok IPv4 metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "10.0.0.1:9999",
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok IPv6 metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "[fd01:2345::1]:9999",
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "ok unspecified wrong-family metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "0.0.0.0:9999",
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong family metricsBindAddress",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
MetricsBindAddress: "10.0.0.1:9999",
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: true,
dsErr: false,
fatal: false,
},
{
name: "ok ipvs.excludeCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
ExcludeCIDRs: []string{"10.0.0.0/8"},
},
},
PrimaryIPFamily: v1.IPv4Protocol,
},
ssErr: false,
dsErr: false,
},
{
name: "wrong family ipvs.excludeCIDRs",
proxy: &ProxyServer{
Config: &kubeproxyconfig.KubeProxyConfiguration{
IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
ExcludeCIDRs: []string{"10.0.0.0/8", "192.168.0.0/24"},
},
},
PrimaryIPFamily: v1.IPv6Protocol,
},
ssErr: true,
dsErr: false,
fatal: false,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
err, fatal := checkIPConfig(c.proxy, false)
if err != nil && !c.ssErr {
t.Errorf("unexpected error in single-stack case: %v", err)
} else if err == nil && c.ssErr {
t.Errorf("unexpected lack of error in single-stack case")
} else if fatal != c.fatal {
t.Errorf("expected fatal=%v, got %v", c.fatal, fatal)
}
err, fatal = checkIPConfig(c.proxy, true)
if err != nil && !c.dsErr {
t.Errorf("unexpected error in dual-stack case: %v", err)
} else if err == nil && c.dsErr {
t.Errorf("unexpected lack of error in dual-stack case")
} else if fatal != c.fatal {
t.Errorf("expected fatal=%v, got %v", c.fatal, fatal)
}
})
}
}