Update cri plugin to v1.0.0-rc.2.

Signed-off-by: Lantao Liu <lantaol@google.com>

.travis.yml

@@ -34,8 +34,6 @@ before_install:
   - uname -r
   - sudo apt-get -q update
   - sudo apt-get install -y libseccomp-dev/trusty-backports
-  # Use jpetazzo/nsenter to install nsenter on ubuntu trusty.
-  - docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
 
 install:
   - sudo PATH=$PATH GOPATH=$GOPATH script/setup/install-protobuf

vendor.conf

@@ -44,7 +44,7 @@ github.com/gotestyourself/gotestyourself 44dbf532bbf5767611f6f2a61bded572e337010
 github.com/google/go-cmp v0.1.0
 
 # cri dependencies
-github.com/containerd/cri v1.0.0-rc.1
+github.com/containerd/cri v1.0.0-rc.2
 github.com/containerd/go-cni f2d7272f12d045b16ed924f50e91f9f9cecc55a7
 github.com/blang/semver v3.1.0
 github.com/containernetworking/cni v0.6.0

vendor/github.com/containerd/cri/README.md

@@ -67,12 +67,10 @@ specifications as appropriate.
 (Fedora, CentOS, RHEL). On releases of Ubuntu <=Trusty and Debian <=jessie a
 backport version of `libseccomp-dev` is required. See [travis.yml](.travis.yml) for an example on trusty.
 * **btrfs development library.** Required by containerd btrfs support. `btrfs-tools`(Ubuntu, Debian) / `btrfs-progs-devel`(Fedora, CentOS, RHEL)
-2. Install other dependencies:
+2. Install **`socat`** (required by portforward).
-* **`nsenter`**: Required by portforward.
+2. Install and setup a go 1.10 development environment.
-* **`socat`**: Required by portforward.
+3. Make a local clone of this repository.
-3. Install and setup a go 1.10 development environment.
+4. Install binary dependencies by running the following command from your cloned `cri/` project directory:
-4. Make a local clone of this repository.
-5. Install binary dependencies by running the following command from your cloned `cri/` project directory:
 ```bash
 # Note: install.deps installs the above mentioned runc, containerd, and CNI
 # binary dependencies. install.deps is only provided for general use and ease of
@@ -130,6 +128,10 @@ See [here](./docs/testing.md) for information about test.
 ## Using crictl
 See [here](./docs/crictl.md) for information about using `crictl` to debug
 pods, containers, and images.
+## Configurations
+See [here](./docs/config.md) for information about how to configure cri plugins
+and [here](https://github.com/containerd/containerd/blob/master/docs/man/containerd-config.1.md)
+for information about how to configure containerd
 ## Documentation
 See [here](./docs) for additional documentation.
 ## Contributing

vendor/github.com/containerd/cri/pkg/config/config.go

@@ -45,6 +45,17 @@ type CniConfig struct {
 	NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"`
 	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
 	NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"`
+	// NetworkPluginConfTemplate is the file path of golang template used to generate
+	// cni config.
+	// When it is set, containerd will get cidr from kubelet to replace {{.PodCIDR}} in
+	// the template, and write the config into NetworkPluginConfDir.
+	// Ideally the cni config should be placed by system admin or cni daemon like calico,
+	// weaveworks etc. However, there are still users using kubenet
+	// (https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet)
+	// today, who don't have a cni daemonset in production. NetworkPluginConfTemplate is
+	// a temporary backward-compatible solution for them.
+	// TODO(random-liu): Deprecate this option when kubenet is deprecated.
+	NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"`
 }
 
 // Mirror contains the config related to the registry mirror
@@ -108,6 +119,7 @@ func DefaultConfig() PluginConfig {
 		CniConfig: CniConfig{
 			NetworkPluginBinDir:       "/opt/cni/bin",
 			NetworkPluginConfDir:      "/etc/cni/net.d",
+			NetworkPluginConfTemplate: "",
 		},
 		ContainerdConfig: ContainerdConfig{
 			Snapshotter: containerd.DefaultSnapshotter,

vendor/github.com/containerd/cri/pkg/containerd/importer/importer.go

@@ -54,13 +54,29 @@ type manifestDotJSON struct {
 	Parent string
 }
 
-// isLayerTar returns true if name is like "deadbeeddeadbeef/layer.tar"
+// isLayerTar returns true if name is like "foobar/layer.tar"
 func isLayerTar(name string) bool {
 	slashes := len(strings.Split(name, "/"))
 	return slashes == 2 && strings.HasSuffix(name, "/layer.tar")
 }
 
-// isDotJSON returns true if name is like "deadbeefdeadbeef.json"
+// followSymlinkLayer returns actual layer name of the symlink layer.
+// It returns "foobar/layer.tar" if the name is like
+// "../foobar/layer.tar", and returns error if the name
+// is not in "../foobar/layer.tar" format.
+func followSymlinkLayer(name string) (string, error) {
+	parts := strings.Split(name, "/")
+	if len(parts) != 3 || parts[0] != ".." {
+		return "", errors.New("invalid symlink layer")
+	}
+	name = strings.TrimPrefix(name, "../")
+	if !isLayerTar(name) {
+		return "", errors.New("invalid layer tar")
+	}
+	return name, nil
+}
+
+// isDotJSON returns true if name is like "foobar.json"
 func isDotJSON(name string) bool {
 	slashes := len(strings.Split(name, "/"))
 	return slashes == 1 && strings.HasSuffix(name, ".json")
@@ -75,7 +91,7 @@ type imageConfig struct {
 // An image MUST have `manifest.json`.
 // `repositories` file in Docker Image Spec v1.0 is not supported (yet).
 // Also, the current implementation assumes the implicit file name convention,
-// which is not explicitly documented in the spec. (e.g. deadbeef/layer.tar)
+// which is not explicitly documented in the spec. (e.g. foobar/layer.tar)
 // It returns a group of image references successfully loaded.
 func Import(ctx context.Context, client *containerd.Client, reader io.Reader) (_ []string, retErr error) {
 	ctx, done, err := client.WithLease(ctx)
@@ -98,8 +114,9 @@ func Import(ctx context.Context, client *containerd.Client, reader io.Reader) (_
 	tr := tar.NewReader(reader)
 	var (
 		mfsts         []manifestDotJSON
-		layers  = make(map[string]ocispec.Descriptor) // key: filename (deadbeeddeadbeef/layer.tar)
+		symlinkLayers = make(map[string]string)             // key: filename (foobar/layer.tar), value: linkname (targetlayerid/layer.tar)
-		configs = make(map[string]imageConfig)        // key: filename (deadbeeddeadbeef.json)
+		layers        = make(map[string]ocispec.Descriptor) // key: filename (foobar/layer.tar)
+		configs       = make(map[string]imageConfig)        // key: filename (foobar.json)
 	)
 	for {
 		hdr, err := tr.Next()
@@ -109,6 +126,14 @@ func Import(ctx context.Context, client *containerd.Client, reader io.Reader) (_
 		if err != nil {
 			return nil, errors.Wrap(err, "get next file")
 		}
+		if hdr.Typeflag == tar.TypeSymlink && isLayerTar(hdr.Name) {
+			linkname, err := followSymlinkLayer(hdr.Linkname)
+			if err != nil {
+				return nil, errors.Wrapf(err, "follow symlink layer from %q to %q", hdr.Name, hdr.Linkname)
+			}
+			symlinkLayers[hdr.Name] = linkname
+			continue
+		}
 		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
 			continue
 		}
@@ -136,6 +161,13 @@ func Import(ctx context.Context, client *containerd.Client, reader io.Reader) (_
 			continue
 		}
 	}
+	for name, linkname := range symlinkLayers {
+		desc, ok := layers[linkname]
+		if !ok {
+			return nil, errors.Errorf("no target for symlink layer from %q to %q", name, linkname)
+		}
+		layers[name] = desc
+	}
 	var refs []string
 	defer func() {
 		if retErr == nil {
@@ -255,7 +287,7 @@ func onUntarManifestJSON(r io.Reader) ([]manifestDotJSON, error) {
 }
 
 func onUntarLayerTar(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (*ocispec.Descriptor, error) {
-	// name is like "deadbeeddeadbeef/layer.tar" ( guaranteed by isLayerTar() )
+	// name is like "foobar/layer.tar" ( guaranteed by isLayerTar() )
 	split := strings.Split(name, "/")
 	// note: split[0] is not expected digest here
 	cw, err := cs.Writer(ctx, "layer-"+split[0], size, "")
@@ -277,7 +309,7 @@ func onUntarDotJSON(ctx context.Context, r io.Reader, cs content.Ingester, name
 	config := imageConfig{}
 	config.desc.MediaType = images.MediaTypeDockerSchema2Config
 	config.desc.Size = size
-	// name is like "deadbeeddeadbeef.json" ( guaranteed by is DotJSON() )
+	// name is like "foobar.json" ( guaranteed by is DotJSON() )
 	split := strings.Split(name, ".")
 	cw, err := cs.Writer(ctx, "config-"+split[0], size, "")
 	if err != nil {

vendor/github.com/containerd/cri/pkg/containerd/resolver/resolver.go

@@ -27,6 +27,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/containerd/containerd/images"
@@ -250,12 +251,12 @@ func (r *containerdResolver) Pusher(ctx context.Context, ref string) (remotes.Pu
 type dockerBase struct {
 	refspec reference.Spec
 	base    []url.URL
-	token   string
 
 	client           *http.Client
 	useBasic         bool
-	username string
+	username, secret string
-	secret   string
+	token            string
+	mu               sync.Mutex
 }
 
 func (r *containerdResolver) base(refspec reference.Spec) (*dockerBase, error) {
@@ -300,6 +301,23 @@ func (r *containerdResolver) base(refspec reference.Spec) (*dockerBase, error) {
 	}, nil
 }
 
+func (r *dockerBase) getToken() string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.token
+}
+
+func (r *dockerBase) setToken(token string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	changed := r.token != token
+	r.token = token
+
+	return changed
+}
+
 func (r *dockerBase) urls(ps ...string) []string {
 	urls := []string{}
 	for _, url := range r.base {
@@ -310,10 +328,11 @@ func (r *dockerBase) urls(ps ...string) []string {
 }
 
 func (r *dockerBase) authorize(req *http.Request) {
+	token := r.getToken()
 	if r.useBasic {
 		req.SetBasicAuth(r.username, r.secret)
-	} else if r.token != "" {
+	} else if token != "" {
-		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.token))
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
 	}
 }
 
@@ -361,7 +380,7 @@ func (r *dockerBase) retryRequest(ctx context.Context, req *http.Request, respon
 		for _, c := range parseAuthHeader(last.Header) {
 			if c.scheme == bearerAuth {
 				if err := invalidAuthorization(c, responses); err != nil {
-					r.token = ""
+					r.setToken("")
 					return nil, err
 				}
 				if err := r.setTokenAuth(ctx, c.parameters); err != nil {
@@ -446,19 +465,22 @@ func (r *dockerBase) setTokenAuth(ctx context.Context, params map[string]string)
 	if len(to.scopes) == 0 {
 		return errors.New("no scope specified for token auth challenge")
 	}
+
+	var token string
 	if r.secret != "" {
 		// Credential information is provided, use oauth POST endpoint
-		r.token, err = r.fetchTokenWithOAuth(ctx, to)
+		token, err = r.fetchTokenWithOAuth(ctx, to)
 		if err != nil {
 			return errors.Wrap(err, "failed to fetch oauth token")
 		}
 	} else {
 		// Do request anonymously
-		r.token, err = r.getToken(ctx, to)
+		token, err = r.fetchToken(ctx, to)
 		if err != nil {
 			return errors.Wrap(err, "failed to fetch anonymous token")
 		}
 	}
+	r.setToken(token)
 
 	return nil
 }
@@ -502,7 +524,7 @@ func (r *dockerBase) fetchTokenWithOAuth(ctx context.Context, to tokenOptions) (
 	// Registries without support for POST may return 404 for POST /v2/token.
 	// As of September 2017, GCR is known to return 404.
 	if (resp.StatusCode == 405 && r.username != "") || resp.StatusCode == 404 || resp.StatusCode == 401 {
-		return r.getToken(ctx, to)
+		return r.fetchToken(ctx, to)
 	} else if resp.StatusCode < 200 || resp.StatusCode >= 400 {
 		b, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 64000)) // 64KB
 		log.G(ctx).WithFields(logrus.Fields{
@@ -531,8 +553,8 @@ type getTokenResponse struct {
 	RefreshToken string    `json:"refresh_token"`
 }
 
-// getToken fetches a token using a GET request
+// fetchToken fetches a token using a GET request
-func (r *dockerBase) getToken(ctx context.Context, to tokenOptions) (string, error) {
+func (r *dockerBase) fetchToken(ctx context.Context, to tokenOptions) (string, error) {
 	req, err := http.NewRequest("GET", to.realm, nil)
 	if err != nil {
 		return "", err

vendor/github.com/containerd/cri/pkg/server/sandbox_portforward.go

@@ -23,18 +23,17 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
-	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
 	sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
 )
 
 // PortForward prepares a streaming endpoint to forward ports from a PodSandbox, and returns the address.
 func (c *criService) PortForward(ctx context.Context, r *runtime.PortForwardRequest) (retRes *runtime.PortForwardResponse, retErr error) {
-	// TODO(random-liu): Run a socat container inside the sandbox to do portforward.
 	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to find sandbox %q", r.GetPodSandboxId())
@@ -46,39 +45,29 @@ func (c *criService) PortForward(ctx context.Context, r *runtime.PortForwardRequ
 	return c.streamServer.GetPortForward(r)
 }
 
-// portForward requires `nsenter` and `socat` on the node, it uses `nsenter` to enter the
+// portForward requires `socat` on the node. It uses netns to enter the sandbox namespace,
-// sandbox namespace, and run `socat` inside the namespace to forward stream for a specific
+// and run `socat` insidethe namespace to forward stream for a specific port. The `socat`
-// port. The `socat` command keeps running until it exits or client disconnect.
+// command keeps running until it exits or client disconnect.
 func (c *criService) portForward(id string, port int32, stream io.ReadWriteCloser) error {
 	s, err := c.sandboxStore.Get(id)
 	if err != nil {
 		return errors.Wrapf(err, "failed to find sandbox %q in store", id)
 	}
-	t, err := s.Container.Task(ctrdutil.NamespacedContext(), nil)
+	if s.NetNS == nil || s.NetNS.Closed() {
-	if err != nil {
+		return errors.Errorf("network namespace for sandbox %q is closed", id)
-		return errors.Wrap(err, "failed to get sandbox container task")
 	}
-	pid := t.Pid()
 
 	socat, err := exec.LookPath("socat")
 	if err != nil {
 		return errors.Wrap(err, "failed to find socat")
 	}
 
-	// Check following links for meaning of the options:
+	// Check https://linux.die.net/man/1/socat for meaning of the options.
-	// * socat: https://linux.die.net/man/1/socat
+	args := []string{socat, "-", fmt.Sprintf("TCP4:localhost:%d", port)}
-	// * nsenter: http://man7.org/linux/man-pages/man1/nsenter.1.html
-	args := []string{"-t", fmt.Sprintf("%d", pid), "-n", socat,
-		"-", fmt.Sprintf("TCP4:localhost:%d", port)}
 
-	nsenter, err := exec.LookPath("nsenter")
+	logrus.Infof("Executing port forwarding command %q in network namespace %q", strings.Join(args, " "), s.NetNS.GetPath())
-	if err != nil {
+	err = s.NetNS.GetNs().Do(func(_ ns.NetNS) error {
-		return errors.Wrap(err, "failed to find nsenter")
+		cmd := exec.Command(args[0], args[1:]...)
-	}
-
-	logrus.Infof("Executing port forwarding command: %s %s", nsenter, strings.Join(args, " "))
-
-	cmd := exec.Command(nsenter, args...)
 		cmd.Stdout = stream
 
 		stderr := new(bytes.Buffer)
@@ -102,13 +91,17 @@ func (c *criService) portForward(id string, port int32, stream io.ReadWriteClose
 				logrus.WithError(err).Errorf("Failed to copy port forward input for %q port %d", id, port)
 			}
 			in.Close()
-		logrus.Debugf("Finish copy port forward input for %q port %d", id, port)
+			logrus.Debugf("Finish copying port forward input for %q port %d", id, port)
 		}()
 
 		if err := cmd.Run(); err != nil {
 			return errors.Errorf("nsenter command returns error: %v, stderr: %q", err, stderr.String())
 		}
-
+		return nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, "failed to execute portforward in network namespace %s", s.NetNS.GetPath())
+	}
 	logrus.Infof("Finish port forwarding for %q port %d", id, port)
 
 	return nil

vendor/github.com/containerd/cri/pkg/server/update_runtime_config.go

@@ -17,13 +17,62 @@ limitations under the License.
 package server
 
 import (
-	"golang.org/x/net/context"
+	"os"
+	"path/filepath"
+	"text/template"
 
+	cni "github.com/containerd/go-cni"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
+// cniConfigTemplate contains the values containerd will overwrite
+// in the cni config template.
+type cniConfigTemplate struct {
+	// PodCIDR is the cidr for pods on the node.
+	PodCIDR string
+}
+
+// cniConfigFileName is the name of cni config file generated by containerd.
+const cniConfigFileName = "10-containerd-net.conflist"
+
 // UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates.
-// TODO(random-liu): Figure out how to handle pod cidr in the cri plugin.
 func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) {
+	podCIDR := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()
+	if podCIDR == "" {
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	confTemplate := c.config.NetworkPluginConfTemplate
+	if confTemplate == "" {
+		logrus.Info("No cni config template is specified, wait for other system components to drop the config.")
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	if err := c.netPlugin.Status(); err == nil {
+		logrus.Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate)
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	} else if err := c.netPlugin.Load(cni.WithLoNetwork(), cni.WithDefaultConf()); err == nil {
+		logrus.Infof("CNI config is successfully loaded, skip generating cni config from template %q", confTemplate)
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	logrus.Infof("Generating cni config from template %q", confTemplate)
+	// generate cni config file from the template with updated pod cidr.
+	t, err := template.ParseFiles(confTemplate)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse cni config template %q", confTemplate)
+	}
+	if err := os.MkdirAll(c.config.NetworkPluginConfDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create cni config directory: %q", c.config.NetworkPluginConfDir)
+	}
+	confFile := filepath.Join(c.config.NetworkPluginConfDir, cniConfigFileName)
+	f, err := os.OpenFile(confFile, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open cni config file %q", confFile)
+	}
+	defer f.Close()
+	if err := t.Execute(f, cniConfigTemplate{PodCIDR: podCIDR}); err != nil {
+		return nil, errors.Wrapf(err, "failed to generate cni config file %q", confFile)
+	}
 	return &runtime.UpdateRuntimeConfigResponse{}, nil
 }

vendor/github.com/containerd/cri/pkg/store/sandbox/netns.go

@@ -117,3 +117,10 @@ func (n *NetNS) GetPath() string {
 	defer n.Unlock()
 	return n.ns.Path()
 }
+
+// GetNs returns the network namespace handle
+func (n *NetNS) GetNs() cnins.NetNS {
+	n.Lock()
+	defer n.Unlock()
+	return n.ns
+}

vendor/github.com/containerd/cri/vendor.conf

@@ -4,7 +4,7 @@ github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/containerd/cgroups fe281dd265766145e943a034aa41086474ea6130
 github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925
-github.com/containerd/containerd d1b3ea406130fdb7284f14a8754b2272f2537c4c
+github.com/containerd/containerd v1.1.0-rc.1
 github.com/containerd/continuity 3e8f2ea4b190484acb976a5b378d373429639a1a
 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
 github.com/containerd/go-runc bcb223a061a3dd7de1a89c0b402a60f4dd9bd307