diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 9c79e6dff34..f96a3f32695 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -20,6 +20,7 @@ import ( "fmt" "io" + "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/componentconfig" internalApi "k8s.io/kubernetes/pkg/kubelet/api" @@ -82,9 +83,10 @@ type NetworkPluginSettings struct { // RuntimeHost is an interface that serves as a trap-door from plugin back // into the kubelet. - // TODO: This shouldn't be required, remove once we move host - // ports into CNI. - RuntimeHost network.Host + // TODO: This shouldn't be required, remove once we move host ports into CNI + // and figure out bandwidth shaping. See corresponding comments above + // network.Host interface. + LegacyRuntimeHost network.LegacyHost } var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPathLabelKey, sandboxIDLabelKey} @@ -113,11 +115,16 @@ func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot str // dockershim currently only supports CNI plugins. cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDir) cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir)) - plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, pluginSettings.RuntimeHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU) + netHost := &dockerNetworkHost{ + pluginSettings.LegacyRuntimeHost, + &namespaceGetter{ds}, + } + plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU) if err != nil { return nil, fmt.Errorf("didn't find compatible CNI plugin with given settings %+v: %v", pluginSettings, err) } ds.networkPlugin = plug + glog.Infof("Docker cri networking managed by %v", plug.Name()) return ds, nil } @@ -182,3 +189,28 @@ func (ds *dockerService) UpdateRuntimeConfig(runtimeConfig *runtimeApi.RuntimeCo } return } + +// namespaceGetter is a wrapper around the dockerService that implements +// the network.NamespaceGetter interface. +type namespaceGetter struct { + *dockerService +} + +// GetNetNS returns the network namespace of the given containerID. The ID +// supplied is typically the ID of a pod sandbox. This getter doesn't try +// to map non-sandbox IDs to their respective sandboxes. +func (ds *dockerService) GetNetNS(podSandboxID string) (string, error) { + r, err := ds.client.InspectContainer(podSandboxID) + if err != nil { + return "", err + } + return getNetworkNamespace(r), nil +} + +// dockerNetworkHost implements network.Host by wrapping the legacy host +// passed in by the kubelet and adding NamespaceGetter methods. The legacy +// host methods are slated for deletion. +type dockerNetworkHost struct { + network.LegacyHost + *namespaceGetter +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 647ae4913d4..4751a8b8996 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -456,7 +456,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub } glog.Infof("Hairpin mode set to %q", klet.hairpinMode) - if plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, kubeCfg.NetworkPluginName, &networkHost{klet}, klet.hairpinMode, klet.nonMasqueradeCIDR, int(kubeCfg.NetworkPluginMTU)); err != nil { + if plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, kubeCfg.NetworkPluginName, &criNetworkHost{&networkHost{klet}}, klet.hairpinMode, klet.nonMasqueradeCIDR, int(kubeCfg.NetworkPluginMTU)); err != nil { return nil, err } else { klet.networkPlugin = plug @@ -482,6 +482,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub } } + // TODO: These need to become arguments to a standalone docker shim. binDir := kubeCfg.CNIBinDir if binDir == "" { binDir = kubeCfg.NetworkPluginDir @@ -493,9 +494,14 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub PluginConfDir: kubeCfg.CNIConfDir, PluginBinDir: binDir, MTU: int(kubeCfg.NetworkPluginMTU), - RuntimeHost: &networkHost{klet}, } + // Remote runtime shim just cannot talk back to kubelet, so it doesn't + // support bandwidth shaping or hostports till #35457. To enable legacy + // features, replace with networkHost. + var nl *noOpLegacyHost + pluginSettings.LegacyRuntimeHost = nl + // Initialize the runtime. switch kubeCfg.ContainerRuntime { case "docker": diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 67f1c47404a..69667df419e 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -193,7 +193,7 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec if err := plugin.checkInitialized(); err != nil { return err } - netnsPath, err := plugin.host.GetRuntime().GetNetNS(id) + netnsPath, err := plugin.host.GetNetNS(id.ID) if err != nil { return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) } @@ -217,7 +217,7 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku if err := plugin.checkInitialized(); err != nil { return err } - netnsPath, err := plugin.host.GetRuntime().GetNetNS(id) + netnsPath, err := plugin.host.GetNetNS(id.ID) if err != nil { return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) } @@ -228,7 +228,7 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku // TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin. // Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) { - netnsPath, err := plugin.host.GetRuntime().GetNetNS(id) + netnsPath, err := plugin.host.GetNetNS(id.ID) if err != nil { return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) } diff --git a/pkg/kubelet/network/kubenet/kubenet_linux.go b/pkg/kubelet/network/kubenet/kubenet_linux.go index 717aefc8e62..f6e75057094 100644 --- a/pkg/kubelet/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/network/kubenet/kubenet_linux.go @@ -533,7 +533,7 @@ func (plugin *kubenetNetworkPlugin) GetPodNetworkStatus(namespace string, name s return &network.PodNetworkStatus{IP: net.ParseIP(podIP)}, nil } - netnsPath, err := plugin.host.GetRuntime().GetNetNS(id) + netnsPath, err := plugin.host.GetNetNS(id.ID) if err != nil { return nil, fmt.Errorf("Kubenet failed to retrieve network namespace path: %v", err) } @@ -722,7 +722,7 @@ func podIsExited(p *kubecontainer.Pod) bool { } func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubecontainer.ContainerID) (*libcni.RuntimeConf, error) { - netnsPath, err := plugin.host.GetRuntime().GetNetNS(id) + netnsPath, err := plugin.host.GetNetNS(id.ID) if err != nil { return nil, fmt.Errorf("Kubenet failed to retrieve network namespace path: %v", err) } diff --git a/pkg/kubelet/network/plugins.go b/pkg/kubelet/network/plugins.go index c5e6ae26c28..9e7a1b65c26 100644 --- a/pkg/kubelet/network/plugins.go +++ b/pkg/kubelet/network/plugins.go @@ -95,16 +95,53 @@ type PodNetworkStatus struct { IP net.IP `json:"ip" description:"Primary IP address of the pod"` } -// Host is an interface that plugins can use to access the kubelet. -type Host interface { +// LegacyHost implements the methods required by network plugins that +// were directly invoked by the kubelet. Implementations of this interface +// that do not wish to support these features can simply return false +// to SupportsLegacyFeatures. +type LegacyHost interface { // Get the pod structure by its name, namespace + // Only used for hostport management and bw shaping GetPodByName(namespace, name string) (*api.Pod, bool) // GetKubeClient returns a client interface + // Only used in testing GetKubeClient() clientset.Interface // GetContainerRuntime returns the container runtime that implements the containers (e.g. docker/rkt) + // Only used for hostport management GetRuntime() kubecontainer.Runtime + + // SupportsLegacyFeaturs returns true if this host can support hostports + // and bandwidth shaping. Both will either get added to CNI or dropped, + // so differnt implementations can choose to ignore them. + SupportsLegacyFeatures() bool +} + +// Host is an interface that plugins can use to access the kubelet. +// TODO(#35457): get rid of this backchannel to the kubelet. The scope of +// the back channel is restricted to host-ports/testing, and restricted +// to kubenet. No other network plugin wrapper needs it. Other plugins +// only require a way to access namespace information, which they can do +// directly through the embedded NamespaceGetter. +type Host interface { + // NamespaceGetter is a getter for sandbox namespace information. + // It's the only part of this interface that isn't currently deprecated. + NamespaceGetter + + // LegacyHost contains methods that trap back into the Kubelet. Dependence + // *do not* add more dependencies in this interface. In a post-cri world, + // network plugins will be invoked by the runtime shim, and should only + // require NamespaceGetter. + LegacyHost +} + +// NamespaceGetter is an interface to retrieve namespace information for a given +// sandboxID. Typically implemented by runtime shims that are closely coupled to +// CNI plugin wrappers like kubenet. +type NamespaceGetter interface { + // GetNetNS returns network namespace information for the given containerID. + GetNetNS(containerID string) (string, error) } // InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names. diff --git a/pkg/kubelet/networks.go b/pkg/kubelet/networks.go index 9ab4a137e6e..cef6703e2a7 100644 --- a/pkg/kubelet/networks.go +++ b/pkg/kubelet/networks.go @@ -24,6 +24,11 @@ import ( // This just exports required functions from kubelet proper, for use by network // plugins. +// TODO(#35457): get rid of this backchannel to the kubelet. The scope of +// the back channel is restricted to host-ports/testing, and restricted +// to kubenet. No other network plugin wrapper needs it. Other plugins +// only require a way to access namespace information, which they can do +// directly through the methods implemented by criNetworkHost. type networkHost struct { kubelet *Kubelet } @@ -39,3 +44,45 @@ func (nh *networkHost) GetKubeClient() clientset.Interface { func (nh *networkHost) GetRuntime() kubecontainer.Runtime { return nh.kubelet.GetRuntime() } + +func (nh *networkHost) SupportsLegacyFeatures() bool { + return true +} + +// criNetworkHost implements the part of network.Host required by the +// cri (NamespaceGetter). It leechs off networkHost for all other +// methods, because networkHost is slated for deletion. +type criNetworkHost struct { + *networkHost +} + +// GetNetNS returns the network namespace of the given containerID. +// This method satisfies the network.NamespaceGetter interface for +// networkHost. It's only meant to be used from network plugins +// that are directly invoked by the kubelet (aka: legacy, pre-cri). +// Any network plugin invoked by a cri must implement NamespaceGetter +// to talk directly to the runtime instead. +func (c *criNetworkHost) GetNetNS(containerID string) (string, error) { + return c.kubelet.GetRuntime().GetNetNS(kubecontainer.ContainerID{"", containerID}) +} + +// noOpLegacyHost implements the network.LegacyHost interface for the remote +// runtime shim by just returning empties. It doesn't support legacy features +// like host port and bandwidth shaping. +type noOpLegacyHost struct{} + +func (n *noOpLegacyHost) GetPodByName(namespace, name string) (*api.Pod, bool) { + return nil, true +} + +func (n *noOpLegacyHost) GetKubeClient() clientset.Interface { + return nil +} + +func (n *noOpLegacyHost) GetRuntime() kubecontainer.Runtime { + return nil +} + +func (nh *noOpLegacyHost) SupportsLegacyFeatures() bool { + return false +} diff --git a/pkg/kubelet/remote/remote_runtime.go b/pkg/kubelet/remote/remote_runtime.go index c7b46603390..17aa3d0f49c 100644 --- a/pkg/kubelet/remote/remote_runtime.go +++ b/pkg/kubelet/remote/remote_runtime.go @@ -36,7 +36,7 @@ type RemoteRuntimeService struct { // NewRemoteRuntimeService creates a new internalApi.RuntimeService. func NewRemoteRuntimeService(addr string, connectionTimout time.Duration) (internalApi.RuntimeService, error) { - glog.V(3).Infof("Connecting to runtime service %s", addr) + glog.Infof("Connecting to runtime service %s", addr) conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimout), grpc.WithDialer(dial)) if err != nil { glog.Errorf("Connect remote runtime %s failed: %v", addr, err)