iptables part implementation

This commit is contained in:
m1093782566 2018-01-17 10:30:19 +08:00
parent bf565305ee
commit ddfa04e8f4
6 changed files with 217 additions and 63 deletions

View File

@ -341,6 +341,12 @@ type Proxier struct {
filterRules *bytes.Buffer
natChains *bytes.Buffer
natRules *bytes.Buffer
// Values are as a parameter to select the interfaces where nodeport works.
nodePortAddresses []string
// networkInterfacer defines an interface for several net library functions.
// Inject for test purpose.
networkInterfacer utilproxy.NetworkInterfacer
}
// listenPortOpener opens ports by calling bind() and listen().
@ -371,6 +377,7 @@ func NewProxier(ipt utiliptables.Interface,
nodeIP net.IP,
recorder record.EventRecorder,
healthzServer healthcheck.HealthzUpdater,
nodePortAddresses []string,
) (*Proxier, error) {
// Set the route_localnet sysctl we need for
if err := sysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil {
@ -422,6 +429,8 @@ func NewProxier(ipt utiliptables.Interface,
filterRules: bytes.NewBuffer(nil),
natChains: bytes.NewBuffer(nil),
natRules: bytes.NewBuffer(nil),
nodePortAddresses: nodePortAddresses,
networkInterfacer: utilproxy.RealNetwork{},
}
burstSyncs := 2
glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs)
@ -1289,11 +1298,37 @@ func (proxier *Proxier) syncProxyRules() {
// Finally, tail-call to the nodeports chain. This needs to be after all
// other service portal rules.
writeLine(proxier.natRules,
"-A", string(kubeServicesChain),
"-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`,
"-m", "addrtype", "--dst-type", "LOCAL",
"-j", string(kubeNodePortsChain))
addresses, err := utilproxy.GetNodeAddresses(proxier.nodePortAddresses, proxier.networkInterfacer)
if err != nil {
glog.Errorf("Failed to get node ip address matching nodeport cidr")
} else {
isIPv6 := proxier.iptables.IsIpv6()
for address := range addresses {
// TODO(thockin, m1093782566): If/when we have dual-stack support we will want to distinguish v4 from v6 zero-CIDRs.
if utilproxy.IsZeroCIDR(address) {
args = append(args[:0],
"-A", string(kubeServicesChain),
"-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`,
"-m", "addrtype", "--dst-type", "LOCAL",
"-j", string(kubeNodePortsChain))
writeLine(proxier.natRules, args...)
// Nothing else matters after the zero CIDR.
break
}
// Ignore IP addresses with incorrect version
if isIPv6 && !conntrack.IsIPv6String(address) || !isIPv6 && conntrack.IsIPv6String(address) {
glog.Errorf("IP address %s has incorrect IP version", address)
continue
}
// create nodeport rules for each IP one by one
args = append(args[:0],
"-A", string(kubeServicesChain),
"-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`,
"-d", address,
"-j", string(kubeNodePortsChain))
writeLine(proxier.natRules, args...)
}
}
// If the masqueradeMark has been added then we want to forward that same
// traffic, this allows NodePort traffic to be forwarded even if the default

View File

@ -34,6 +34,7 @@ import (
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/proxy"
utilproxy "k8s.io/kubernetes/pkg/proxy/util"
utilproxytest "k8s.io/kubernetes/pkg/proxy/util/testing"
"k8s.io/kubernetes/pkg/util/async"
"k8s.io/kubernetes/pkg/util/conntrack"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
@ -405,6 +406,8 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier {
filterRules: bytes.NewBuffer(nil),
natChains: bytes.NewBuffer(nil),
natRules: bytes.NewBuffer(nil),
nodePortAddresses: make([]string, 0),
networkInterfacer: utilproxytest.NewFakeNetwork(),
}
p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1)
return p
@ -769,6 +772,14 @@ func TestNodePort(t *testing.T) {
}),
)
itf := net.Interface{Index: 0, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}
addrs := []net.Addr{utilproxytest.AddrStruct{Val: "127.0.0.1/16"}}
itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}
addrs1 := []net.Addr{utilproxytest.AddrStruct{Val: "::1/128"}}
fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf, addrs)
fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1)
fp.nodePortAddresses = []string{}
fp.syncProxyRules()
proto := strings.ToLower(string(api.ProtocolTCP))
@ -1001,6 +1012,11 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable
}),
)
itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}
addrs := []net.Addr{utilproxytest.AddrStruct{Val: "10.20.30.51/24"}}
fp.networkInterfacer.(*utilproxytest.FakeNetwork).AddInterfaceAddr(&itf, addrs)
fp.nodePortAddresses = []string{"10.20.30.0/24"}
fp.syncProxyRules()
proto := strings.ToLower(string(api.ProtocolTCP))
@ -1013,11 +1029,15 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable
if !hasJump(kubeNodePortRules, lbChain, "", svcNodePort) {
errorf(fmt.Sprintf("Failed to find jump to lb chain %v", lbChain), kubeNodePortRules, t)
}
if !hasJump(kubeNodePortRules, string(KubeMarkMasqChain), "", svcNodePort) {
errorf(fmt.Sprintf("Failed to find jump to %s chain for destination IP %d", KubeMarkMasqChain, svcNodePort), kubeNodePortRules, t)
}
kubeServiceRules := ipt.GetRules(string(kubeServicesChain))
if !hasJump(kubeServiceRules, string(kubeNodePortsChain), "10.20.30.51", 0) {
errorf(fmt.Sprintf("Failed to find jump to KUBE-NODEPORTS chain %v", string(kubeNodePortsChain)), kubeServiceRules, t)
}
svcChain := string(servicePortChainName(svcPortName.String(), proto))
lbRules := ipt.GetRules(lbChain)
if hasJump(lbRules, nonLocalEpChain, "", 0) {

45
pkg/proxy/util/network.go Normal file
View File

@ -0,0 +1,45 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"net"
)
// NetworkInterfacer defines an interface for several net library functions. Production
// code will forward to net library functions, and unit tests will override the methods
// for testing purposes.
type NetworkInterfacer interface {
Addrs(intf *net.Interface) ([]net.Addr, error)
Interfaces() ([]net.Interface, error)
}
// RealNetwork implements the NetworkInterfacer interface for production code, just
// wrapping the underlying net library function calls.
type RealNetwork struct{}
// Addrs wraps net.Interface.Addrs(), it's a part of NetworkInterfacer interface.
func (_ RealNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) {
return intf.Addrs()
}
// Interfaces wraps net.Interfaces(), it's a part of NetworkInterfacer interface.
func (_ RealNetwork) Interfaces() ([]net.Interface, error) {
return net.Interfaces()
}
var _ NetworkInterfacer = &RealNetwork{}

View File

@ -0,0 +1,64 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testing
import "net"
// FakeNetwork implements the NetworkInterfacer interface for test purpose.
type FakeNetwork struct {
NetworkInterfaces []net.Interface
// The key of map Addrs is the network interface name
Address map[string][]net.Addr
}
func NewFakeNetwork() *FakeNetwork {
return &FakeNetwork{
NetworkInterfaces: make([]net.Interface, 0),
Address: make(map[string][]net.Addr),
}
}
// AddInterfaceAddr create an interface and its associated addresses for FakeNetwork implementation.
func (f *FakeNetwork) AddInterfaceAddr(intf *net.Interface, addrs []net.Addr) {
f.NetworkInterfaces = append(f.NetworkInterfaces, *intf)
f.Address[intf.Name] = addrs
}
// Addrs is part of NetworkInterfacer interface.
func (f *FakeNetwork) Addrs(intf *net.Interface) ([]net.Addr, error) {
return f.Address[intf.Name], nil
}
// Interfaces is part of NetworkInterfacer interface.
func (f *FakeNetwork) Interfaces() ([]net.Interface, error) {
return f.NetworkInterfaces, nil
}
// AddrStruct implements the net.Addr for test purpose.
type AddrStruct struct{ Val string }
// Network is part of net.Addr interface.
func (a AddrStruct) Network() string {
return a.Val
}
// String is part of net.Addr interface.
func (a AddrStruct) String() string {
return a.Val
}
var _ net.Addr = &AddrStruct{}

View File

@ -21,11 +21,9 @@ import (
"net"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/core/helper"
"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
"k8s.io/kubernetes/pkg/util/conntrack"
"github.com/golang/glog"
@ -74,11 +72,15 @@ func ShouldSkipService(svcName types.NamespacedName, service *api.Service) bool
return false
}
// GetNodeAddresses list all match node IP addresses based on given cidr list. Inject NetworkInterface for test purpose.
// We expect the cidr list passed in is already validated.
// Given `default-route`, it will the IP of host interface having default route.
// GetNodeAddresses return all matched node IP addresses based on given cidr slice.
// Some callers, e.g. IPVS proxier, need concrete IPs, not ranges, which is why this exists.
// NetworkInterfacer is injected for test purpose.
// We expect the cidrs passed in is already validated.
// Given an empty input `[]`, it will return `0.0.0.0/0` and `::/0` directly.
func GetNodeAddresses(cidrs []string, nw NetworkInterface) (sets.String, error) {
// If multiple cidrs is given, it will return the minimal IP sets, e.g. given input `[1.2.0.0/16, 0.0.0.0/0]`, it will
// only return `0.0.0.0/0`.
// NOTE: GetNodeAddresses only accepts CIDRs, if you want concrete IPs, e.g. 1.2.3.4, then the input should be 1.2.3.4/32.
func GetNodeAddresses(cidrs []string, nw NetworkInterfacer) (sets.String, error) {
uniqueAddressList := sets.NewString()
if len(cidrs) == 0 {
uniqueAddressList.Insert(IPv4ZeroCIDR)
@ -96,19 +98,6 @@ func GetNodeAddresses(cidrs []string, nw NetworkInterface) (sets.String, error)
if IsZeroCIDR(cidr) {
continue
}
if cidr == string(kubeproxyconfig.DefaultRoute) {
hostIP, err := utilnet.ChooseHostInterface()
if err != nil {
return nil, fmt.Errorf("error selecting a host interface having default route, error: %v", err)
}
if conntrack.IsIPv6(hostIP) && !uniqueAddressList.Has(IPv6ZeroCIDR) {
uniqueAddressList.Insert(hostIP.String())
}
if !conntrack.IsIPv6(hostIP) && !uniqueAddressList.Has(IPv4ZeroCIDR) {
uniqueAddressList.Insert(hostIP.String())
}
continue
}
_, ipNet, _ := net.ParseCIDR(cidr)
itfs, err := nw.Interfaces()
if err != nil {

View File

@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
api "k8s.io/kubernetes/pkg/apis/core"
fake "k8s.io/kubernetes/pkg/proxy/util/testing"
)
func TestShouldSkipService(t *testing.T) {
@ -120,193 +121,193 @@ type InterfaceAddrsPair struct {
func TestGetNodeAddressses(t *testing.T) {
testCases := []struct {
cidrs []string
nw *FakeNetwork
nw *fake.FakeNetwork
itfAddrsPairs []InterfaceAddrsPair
expected sets.String
}{
{ // case 0
cidrs: []string{"10.20.30.0/24"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}},
},
{
itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "100.200.201.1/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "100.200.201.1/24"}},
},
},
expected: sets.NewString("10.20.30.51"),
},
{ // case 1
cidrs: []string{"0.0.0.0/0"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}},
addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}},
},
},
expected: sets.NewString("0.0.0.0/0"),
},
{ // case 2
cidrs: []string{"2001:db8::/32", "::1/128"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}},
addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "::1/128"}},
addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}},
},
},
expected: sets.NewString("2001:db8::1", "::1"),
},
{ // case 3
cidrs: []string{"::/0"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}},
addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "::1/128"}},
addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}},
},
},
expected: sets.NewString("::/0"),
},
{ // case 4
cidrs: []string{"127.0.0.1/32"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}},
addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}},
},
},
expected: sets.NewString("127.0.0.1"),
},
{ // case 5
cidrs: []string{"127.0.0.0/8"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "127.0.1.1/8"}},
addrs: []net.Addr{fake.AddrStruct{Val: "127.0.1.1/8"}},
},
},
expected: sets.NewString("127.0.1.1"),
},
{ // case 6
cidrs: []string{"10.20.30.0/24", "100.200.201.0/24"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "10.20.30.51/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "10.20.30.51/24"}},
},
{
itf: net.Interface{Index: 2, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "100.200.201.1/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "100.200.201.1/24"}},
},
},
expected: sets.NewString("10.20.30.51", "100.200.201.1"),
},
{ // case 7
cidrs: []string{"10.20.30.0/24", "100.200.201.0/24"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "192.168.1.2/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "192.168.1.2/24"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}},
addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}},
},
},
expected: sets.NewString(),
},
{ // case 8
cidrs: []string{},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "192.168.1.2/24"}},
addrs: []net.Addr{fake.AddrStruct{Val: "192.168.1.2/24"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "127.0.0.1/8"}},
addrs: []net.Addr{fake.AddrStruct{Val: "127.0.0.1/8"}},
},
},
expected: sets.NewString("0.0.0.0/0", "::/0"),
},
{ // case 9
cidrs: []string{},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "2001:db8::1/32"}},
addrs: []net.Addr{fake.AddrStruct{Val: "2001:db8::1/32"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "::1/128"}},
addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}},
},
},
expected: sets.NewString("0.0.0.0/0", "::/0"),
},
{ // case 9
cidrs: []string{"1.2.3.0/24", "0.0.0.0/0"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}},
addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}},
},
},
expected: sets.NewString("0.0.0.0/0"),
},
{ // case 10
cidrs: []string{"0.0.0.0/0", "1.2.3.0/24", "::1/128"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}},
addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "::1/128"}},
addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}},
},
},
expected: sets.NewString("0.0.0.0/0", "::1"),
},
{ // case 11
cidrs: []string{"::/0", "1.2.3.0/24", "::1/128"},
nw: NewFakeNetwork(),
nw: fake.NewFakeNetwork(),
itfAddrsPairs: []InterfaceAddrsPair{
{
itf: net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "1.2.3.4/30"}},
addrs: []net.Addr{fake.AddrStruct{Val: "1.2.3.4/30"}},
},
{
itf: net.Interface{Index: 1, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0},
addrs: []net.Addr{AddrStruct{Val: "::1/128"}},
addrs: []net.Addr{fake.AddrStruct{Val: "::1/128"}},
},
},
expected: sets.NewString("::/0", "1.2.3.4"),