Split network.Host into LegacyHost and NamespaceGetter

pkg/kubelet/dockershim/docker_service.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	internalApi "k8s.io/kubernetes/pkg/kubelet/api"
@@ -82,9 +83,10 @@ type NetworkPluginSettings struct {
 
 	// RuntimeHost is an interface that serves as a trap-door from plugin back
 	// into the kubelet.
-	// TODO: This shouldn't be required, remove once we move host
-	// ports into CNI.
-	RuntimeHost network.Host
+	// TODO: This shouldn't be required, remove once we move host ports into CNI
+	// and figure out bandwidth shaping. See corresponding comments above
+	// network.Host interface.
+	LegacyRuntimeHost network.LegacyHost
 }
 
 var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPathLabelKey, sandboxIDLabelKey}
@@ -113,11 +115,16 @@ func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot str
 	// dockershim currently only supports CNI plugins.
 	cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDir)
 	cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir))
-	plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, pluginSettings.RuntimeHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU)
+	netHost := &dockerNetworkHost{
+		pluginSettings.LegacyRuntimeHost,
+		&namespaceGetter{ds},
+	}
+	plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU)
 	if err != nil {
 		return nil, fmt.Errorf("didn't find compatible CNI plugin with given settings %+v: %v", pluginSettings, err)
 	}
 	ds.networkPlugin = plug
+	glog.Infof("Docker cri networking managed by %v", plug.Name())
 	return ds, nil
 }
 
@@ -182,3 +189,28 @@ func (ds *dockerService) UpdateRuntimeConfig(runtimeConfig *runtimeApi.RuntimeCo
 	}
 	return
 }
+
+// namespaceGetter is a wrapper around the dockerService that implements
+// the network.NamespaceGetter interface.
+type namespaceGetter struct {
+	*dockerService
+}
+
+// GetNetNS returns the network namespace of the given containerID. The ID
+// supplied is typically the ID of a pod sandbox. This getter doesn't try
+// to map non-sandbox IDs to their respective sandboxes.
+func (ds *dockerService) GetNetNS(podSandboxID string) (string, error) {
+	r, err := ds.client.InspectContainer(podSandboxID)
+	if err != nil {
+		return "", err
+	}
+	return getNetworkNamespace(r), nil
+}
+
+// dockerNetworkHost implements network.Host by wrapping the legacy host
+// passed in by the kubelet and adding NamespaceGetter methods. The legacy
+// host methods are slated for deletion.
+type dockerNetworkHost struct {
+	network.LegacyHost
+	*namespaceGetter
+}

pkg/kubelet/kubelet.go

@@ -456,7 +456,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 	}
 	glog.Infof("Hairpin mode set to %q", klet.hairpinMode)
 
-	if plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, kubeCfg.NetworkPluginName, &networkHost{klet}, klet.hairpinMode, klet.nonMasqueradeCIDR, int(kubeCfg.NetworkPluginMTU)); err != nil {
+	if plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, kubeCfg.NetworkPluginName, &criNetworkHost{&networkHost{klet}}, klet.hairpinMode, klet.nonMasqueradeCIDR, int(kubeCfg.NetworkPluginMTU)); err != nil {
 		return nil, err
 	} else {
 		klet.networkPlugin = plug
@@ -482,6 +482,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		}
 	}
 
+	// TODO: These need to become arguments to a standalone docker shim.
 	binDir := kubeCfg.CNIBinDir
 	if binDir == "" {
 		binDir = kubeCfg.NetworkPluginDir
@@ -493,9 +494,14 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		PluginConfDir:     kubeCfg.CNIConfDir,
 		PluginBinDir:      binDir,
 		MTU:               int(kubeCfg.NetworkPluginMTU),
-		RuntimeHost:       &networkHost{klet},
 	}
 
+	// Remote runtime shim just cannot talk back to kubelet, so it doesn't
+	// support bandwidth shaping or hostports till #35457. To enable legacy
+	// features, replace with networkHost.
+	var nl *noOpLegacyHost
+	pluginSettings.LegacyRuntimeHost = nl
+
 	// Initialize the runtime.
 	switch kubeCfg.ContainerRuntime {
 	case "docker":

pkg/kubelet/network/cni/cni.go

@@ -193,7 +193,7 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec
 	if err := plugin.checkInitialized(); err != nil {
 		return err
 	}
-	netnsPath, err := plugin.host.GetRuntime().GetNetNS(id)
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
 	if err != nil {
 		return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
 	}
@@ -217,7 +217,7 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku
 	if err := plugin.checkInitialized(); err != nil {
 		return err
 	}
-	netnsPath, err := plugin.host.GetRuntime().GetNetNS(id)
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
 	if err != nil {
 		return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
 	}
@@ -228,7 +228,7 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku
 // TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin.
 // Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls
 func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
-	netnsPath, err := plugin.host.GetRuntime().GetNetNS(id)
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
 	if err != nil {
 		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
 	}

pkg/kubelet/network/kubenet/kubenet_linux.go

@@ -533,7 +533,7 @@ func (plugin *kubenetNetworkPlugin) GetPodNetworkStatus(namespace string, name s
 		return &network.PodNetworkStatus{IP: net.ParseIP(podIP)}, nil
 	}
 
-	netnsPath, err := plugin.host.GetRuntime().GetNetNS(id)
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
 	if err != nil {
 		return nil, fmt.Errorf("Kubenet failed to retrieve network namespace path: %v", err)
 	}
@@ -722,7 +722,7 @@ func podIsExited(p *kubecontainer.Pod) bool {
 }
 
 func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubecontainer.ContainerID) (*libcni.RuntimeConf, error) {
-	netnsPath, err := plugin.host.GetRuntime().GetNetNS(id)
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
 	if err != nil {
 		return nil, fmt.Errorf("Kubenet failed to retrieve network namespace path: %v", err)
 	}

pkg/kubelet/network/plugins.go

@@ -95,16 +95,53 @@ type PodNetworkStatus struct {
 	IP net.IP `json:"ip" description:"Primary IP address of the pod"`
 }
 
-// Host is an interface that plugins can use to access the kubelet.
-type Host interface {
+// LegacyHost implements the methods required by network plugins that
+// were directly invoked by the kubelet. Implementations of this interface
+// that do not wish to support these features can simply return false
+// to SupportsLegacyFeatures.
+type LegacyHost interface {
 	// Get the pod structure by its name, namespace
+	// Only used for hostport management and bw shaping
 	GetPodByName(namespace, name string) (*api.Pod, bool)
 
 	// GetKubeClient returns a client interface
+	// Only used in testing
 	GetKubeClient() clientset.Interface
 
 	// GetContainerRuntime returns the container runtime that implements the containers (e.g. docker/rkt)
+	// Only used for hostport management
 	GetRuntime() kubecontainer.Runtime
+
+	// SupportsLegacyFeaturs returns true if this host can support hostports
+	// and bandwidth shaping. Both will either get added to CNI or dropped,
+	// so differnt implementations can choose to ignore them.
+	SupportsLegacyFeatures() bool
+}
+
+// Host is an interface that plugins can use to access the kubelet.
+// TODO(#35457): get rid of this backchannel to the kubelet. The scope of
+// the back channel is restricted to host-ports/testing, and restricted
+// to kubenet. No other network plugin wrapper needs it. Other plugins
+// only require a way to access namespace information, which they can do
+// directly through the embedded NamespaceGetter.
+type Host interface {
+	// NamespaceGetter is a getter for sandbox namespace information.
+	// It's the only part of this interface that isn't currently deprecated.
+	NamespaceGetter
+
+	// LegacyHost contains methods that trap back into the Kubelet. Dependence
+	// *do not* add more dependencies in this interface. In a post-cri world,
+	// network plugins will be invoked by the runtime shim, and should only
+	// require NamespaceGetter.
+	LegacyHost
+}
+
+// NamespaceGetter is an interface to retrieve namespace information for a given
+// sandboxID. Typically implemented by runtime shims that are closely coupled to
+// CNI plugin wrappers like kubenet.
+type NamespaceGetter interface {
+	// GetNetNS returns network namespace information for the given containerID.
+	GetNetNS(containerID string) (string, error)
 }
 
 // InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names.

pkg/kubelet/networks.go

@@ -24,6 +24,11 @@ import (
 
 // This just exports required functions from kubelet proper, for use by network
 // plugins.
+// TODO(#35457): get rid of this backchannel to the kubelet. The scope of
+// the back channel is restricted to host-ports/testing, and restricted
+// to kubenet. No other network plugin wrapper needs it. Other plugins
+// only require a way to access namespace information, which they can do
+// directly through the methods implemented by criNetworkHost.
 type networkHost struct {
 	kubelet *Kubelet
 }
@@ -39,3 +44,45 @@ func (nh *networkHost) GetKubeClient() clientset.Interface {
 func (nh *networkHost) GetRuntime() kubecontainer.Runtime {
 	return nh.kubelet.GetRuntime()
 }
+
+func (nh *networkHost) SupportsLegacyFeatures() bool {
+	return true
+}
+
+// criNetworkHost implements the part of network.Host required by the
+// cri (NamespaceGetter). It leechs off networkHost for all other
+// methods, because networkHost is slated for deletion.
+type criNetworkHost struct {
+	*networkHost
+}
+
+// GetNetNS returns the network namespace of the given containerID.
+// This method satisfies the network.NamespaceGetter interface for
+// networkHost. It's only meant to be used from network plugins
+// that are directly invoked by the kubelet (aka: legacy, pre-cri).
+// Any network plugin invoked by a cri must implement NamespaceGetter
+// to talk directly to the runtime instead.
+func (c *criNetworkHost) GetNetNS(containerID string) (string, error) {
+	return c.kubelet.GetRuntime().GetNetNS(kubecontainer.ContainerID{"", containerID})
+}
+
+// noOpLegacyHost implements the network.LegacyHost interface for the remote
+// runtime shim by just returning empties. It doesn't support legacy features
+// like host port and bandwidth shaping.
+type noOpLegacyHost struct{}
+
+func (n *noOpLegacyHost) GetPodByName(namespace, name string) (*api.Pod, bool) {
+	return nil, true
+}
+
+func (n *noOpLegacyHost) GetKubeClient() clientset.Interface {
+	return nil
+}
+
+func (n *noOpLegacyHost) GetRuntime() kubecontainer.Runtime {
+	return nil
+}
+
+func (nh *noOpLegacyHost) SupportsLegacyFeatures() bool {
+	return false
+}

pkg/kubelet/remote/remote_runtime.go

@@ -36,7 +36,7 @@ type RemoteRuntimeService struct {
 
 // NewRemoteRuntimeService creates a new internalApi.RuntimeService.
 func NewRemoteRuntimeService(addr string, connectionTimout time.Duration) (internalApi.RuntimeService, error) {
-	glog.V(3).Infof("Connecting to runtime service %s", addr)
+	glog.Infof("Connecting to runtime service %s", addr)
 	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimout), grpc.WithDialer(dial))
 	if err != nil {
 		glog.Errorf("Connect remote runtime %s failed: %v", addr, err)