Add registry auth config, and use docker resolver in containerd.

Signed-off-by: Lantao Liu <lantaol@google.com>

docs/installation.md

@@ -1,4 +1,3 @@
-<!-- TODO(now) -->
 # Install Containerd with Release Tarball
 This document provides the steps to install `containerd` and its dependencies with the release tarball, and bring up a Kubernetes cluster using kubeadm.
 

docs/registry.md

@@ -1,6 +1,7 @@
 # Configure Image Registry
 This document describes the method to configure the image registry for `containerd` for use with the `cri` plugin.
 
+## Configure Registry Endpoint
 With containerd, `docker.io` is the default image registry. You can also set up other image registries similar to docker.
 
 To configure image registries create/modify the `/etc/containerd/config.toml` as follows:
@@ -19,4 +20,26 @@ The default configuration can be generated by `containerd config default > /etc/
 The endpoint is a list that can contain multiple image registry URLs split by commas. When pulling an image
 from a registry, containerd will try these endpoint URLs one by one, and use the first working one.
 
-After modify the config file, you need restart the `containerd` service.
+After modify this config, you need restart the `containerd` service.
+
+## Configure Registry Credentials
+
+`cri` plugin also supports docker like registry credential config.
+
+To configure a credential for a specific registry endpoint, create/modify the
+`/etc/containerd/config.toml` as follows:
+```toml
+[plugins.cri.registry.auths]
+  [plugins.cri.registry.auths."https://gcr.io"]
+    username = ""
+    password = ""
+    auth = ""
+    identitytoken = ""
+```
+The meaning of each field is the same with the corresponding field in `.docker/config.json`.
+
+Please note that auth config passed by CRI takes precedence over this config.
+The registry credential in this config will only be used when auth config is
+not specified by Kubernetes via CRI.
+
+After modify this config, you need restart the `containerd` service.

hack/verify-lint.sh

@@ -20,7 +20,7 @@ set -o pipefail
 for d in $(find . -type d -a \( -iwholename './pkg*' -o -iwholename './cmd*' \) -not -iwholename './pkg/api*'); do
 	echo for directory ${d} ...
 	gometalinter \
-		 --exclude='error return value not checked.*(Close|Log|Print).*\(errcheck\)$' \
+		 --exclude='error return value not checked.*(Close|Log|Print|Fprint).*\(errcheck\)$' \
 		 --exclude='.*_test\.go:.*error return value not checked.*\(errcheck\)$' \
 		 --exclude='duplicate of.*_test.go.*\(dupl\)$' \
 		 --exclude='.*/mock_.*\.go:.*\(golint\)$' \

pkg/config/config.go

@@ -61,16 +61,32 @@ type CniConfig struct {
 // Mirror contains the config related to the registry mirror
 type Mirror struct {
 	// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
-	// one by one until a working one is found.
+	// one by one until a working one is found. The endpoint must be a valid url
+	// with host specified.
 	Endpoints []string `toml:"endpoint" json:"endpoint"`
-	// TODO (Abhi) We might need to add auth per namespace. Looks like
-	// image auth information is passed by kube itself.
+}
+
+// AuthConfig contains the config related to authentication to a specific registry
+type AuthConfig struct {
+	// Username is the username to login the registry.
+	Username string `toml:"username" json:"username"`
+	// Password is the password to login the registry.
+	Password string `toml:"password" json:"password"`
+	// Auth is a base64 encoded string from the concatenation of the username,
+	// a colon, and the password.
+	Auth string `toml:"auth" json:"auth"`
+	// IdentityToken is used to authenticate the user and get
+	// an access token for the registry.
+	IdentityToken string `toml:"identitytoken" json:"identitytoken"`
 }
 
 // Registry is registry settings configured
 type Registry struct {
 	// Mirrors are namespace to mirror mapping for all namespaces.
 	Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"`
+	// Auths are registry endpoint to auth config mapping. The registry endpoint must
+	// be a valid url with host specified.
+	Auths map[string]AuthConfig `toml:"auths" json:"auths"`
 }
 
 // PluginConfig contains toml config related to CRI plugin,
@@ -81,7 +97,7 @@ type PluginConfig struct {
 	// CniConfig contains config related to cni
 	CniConfig `toml:"cni" json:"cni"`
 	// Registry contains config related to the registry
-	Registry `toml:"registry" json:"registry"`
+	Registry Registry `toml:"registry" json:"registry"`
 	// StreamServerAddress is the ip address streaming server is listening on.
 	StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
 	// StreamServerPort is the port streaming server is listening on.

pkg/server/helpers.go

@@ -444,3 +444,13 @@ func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtim
 	r.Root = runtimeOpts.RuntimeRoot
 	return r, nil
 }
+
+// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
+func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
+	return &runtime.AuthConfig{
+		Username:      a.Username,
+		Password:      a.Password,
+		Auth:          a.Auth,
+		IdentityToken: a.IdentityToken,
+	}
+}

pkg/server/image_pull.go

@@ -19,18 +19,21 @@ package server
 import (
 	"encoding/base64"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
-	containerdresolver "github.com/containerd/cri/pkg/containerd/resolver"
 	imagestore "github.com/containerd/cri/pkg/store/image"
 	"github.com/containerd/cri/pkg/util"
 )
@@ -87,12 +90,7 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
 	if ref != imageRef {
 		logrus.Debugf("PullImage using normalized image ref: %q", ref)
 	}
-	resolver := containerdresolver.NewResolver(containerdresolver.Options{
-		Credentials: func(string) (string, string, error) { return ParseAuth(r.GetAuth()) },
-		Client:      http.DefaultClient,
-		Registry:    c.getResolverOptions(),
-	})
-	_, desc, err := resolver.Resolve(ctx, ref)
+	resolver, desc, err := c.getResolver(ctx, ref, c.credentials(r.GetAuth()))
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to resolve image %q", ref)
 	}
@@ -206,10 +204,59 @@ func (c *criService) createImageReference(ctx context.Context, name string, desc
 	return err
 }
 
-func (c *criService) getResolverOptions() map[string][]string {
-	options := make(map[string][]string)
-	for ns, mirror := range c.config.Mirrors {
-		options[ns] = append(options[ns], mirror.Endpoints...)
+// credentials returns a credential function for docker resolver to use.
+func (c *criService) credentials(auth *runtime.AuthConfig) func(string) (string, string, error) {
+	return func(host string) (string, string, error) {
+		if auth == nil {
+			// Get default auth from config.
+			for h, ac := range c.config.Registry.Auths {
+				u, err := url.Parse(h)
+				if err != nil {
+					return "", "", errors.Wrapf(err, "parse auth host %q", h)
+				}
+				if u.Host == host {
+					auth = toRuntimeAuthConfig(ac)
+					break
+				}
+			}
+		}
+		return ParseAuth(auth)
 	}
-	return options
+}
+
+// getResolver tries registry mirrors and the default registry, and returns the resolver and descriptor
+// from the first working registry.
+func (c *criService) getResolver(ctx context.Context, ref string, cred func(string) (string, string, error)) (remotes.Resolver, imagespec.Descriptor, error) {
+	refspec, err := reference.Parse(ref)
+	if err != nil {
+		return nil, imagespec.Descriptor{}, errors.Wrap(err, "parse image reference")
+	}
+	// Try mirrors in order first, and then try default host name.
+	for _, e := range c.config.Registry.Mirrors[refspec.Hostname()].Endpoints {
+		u, err := url.Parse(e)
+		if err != nil {
+			return nil, imagespec.Descriptor{}, errors.Wrapf(err, "parse registry endpoint %q", e)
+		}
+		resolver := docker.NewResolver(docker.ResolverOptions{
+			Credentials: cred,
+			Client:      http.DefaultClient,
+			Host:        func(string) (string, error) { return u.Host, nil },
+			// By default use "https".
+			PlainHTTP: u.Scheme == "http",
+		})
+		_, desc, err := resolver.Resolve(ctx, ref)
+		if err == nil {
+			return resolver, desc, nil
+		}
+		// Continue to try next endpoint
+	}
+	resolver := docker.NewResolver(docker.ResolverOptions{
+		Credentials: cred,
+		Client:      http.DefaultClient,
+	})
+	_, desc, err := resolver.Resolve(ctx, ref)
+	if err != nil {
+		return nil, imagespec.Descriptor{}, errors.Wrap(err, "no available registry endpoint")
+	}
+	return resolver, desc, nil
 }

pkg/server/image_pull_test.go

@@ -22,6 +22,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/cri/pkg/config"
 )
 
 func TestParseAuth(t *testing.T) {
@@ -72,3 +74,58 @@ func TestParseAuth(t *testing.T) {
 		assert.Equal(t, test.expectedSecret, s)
 	}
 }
+
+func TestCredentials(t *testing.T) {
+	c := newTestCRIService()
+	c.config.Registry.Auths = map[string]criconfig.AuthConfig{
+		"https://test1.io": {
+			Username: "username1",
+			Password: "password1",
+		},
+		"http://test2.io": {
+			Username: "username2",
+			Password: "password2",
+		},
+		"//test3.io": {
+			Username: "username3",
+			Password: "password3",
+		},
+	}
+	for desc, test := range map[string]struct {
+		auth             *runtime.AuthConfig
+		host             string
+		expectedUsername string
+		expectedPassword string
+	}{
+		"auth config from CRI should take precedence": {
+			auth: &runtime.AuthConfig{
+				Username: "username",
+				Password: "password",
+			},
+			host:             "test1.io",
+			expectedUsername: "username",
+			expectedPassword: "password",
+		},
+		"should support https host": {
+			host:             "test1.io",
+			expectedUsername: "username1",
+			expectedPassword: "password1",
+		},
+		"should support http host": {
+			host:             "test2.io",
+			expectedUsername: "username2",
+			expectedPassword: "password2",
+		},
+		"should support hostname only": {
+			host:             "test3.io",
+			expectedUsername: "username3",
+			expectedPassword: "password3",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		username, password, err := c.credentials(test.auth)(test.host)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expectedUsername, username)
+		assert.Equal(t, test.expectedPassword, password)
+	}
+}

vendor/github.com/containerd/containerd/remotes/docker/resolver.go

@@ -53,6 +53,7 @@ var (
 
 type dockerResolver struct {
 	credentials func(string) (string, string, error)
+	host        func(string) (string, error)
 	plainHTTP   bool
 	client      *http.Client
 	tracker     StatusTracker
@@ -65,6 +66,9 @@ type ResolverOptions struct {
 	// is interpretted as a long lived token.
 	Credentials func(string) (string, string, error)
 
+	// Host provides the hostname given a namespace.
+	Host func(string) (string, error)
+
 	// PlainHTTP specifies to use plain http and not https
 	PlainHTTP bool
 
@@ -77,14 +81,27 @@ type ResolverOptions struct {
 	Tracker StatusTracker
 }
 
+// DefaultHost is the default host function.
+func DefaultHost(ns string) (string, error) {
+	if ns == "docker.io" {
+		return "registry-1.docker.io", nil
+	}
+	return ns, nil
+}
+
 // NewResolver returns a new resolver to a Docker registry
 func NewResolver(options ResolverOptions) remotes.Resolver {
 	tracker := options.Tracker
 	if tracker == nil {
 		tracker = NewInMemoryTracker()
 	}
+	host := options.Host
+	if host == nil {
+		host = DefaultHost
+	}
 	return &dockerResolver{
 		credentials: options.Credentials,
+		host:        host,
 		plainHTTP:   options.PlainHTTP,
 		client:      options.Client,
 		tracker:     tracker,
@@ -270,18 +287,19 @@ func (r *dockerResolver) base(refspec reference.Spec) (*dockerBase, error) {
 	)
 
 	host := refspec.Hostname()
-	base.Scheme = "https"
-
-	if host == "docker.io" {
-		base.Host = "registry-1.docker.io"
-	} else {
 	base.Host = host
-
-		if r.plainHTTP || strings.HasPrefix(host, "localhost:") {
-			base.Scheme = "http"
+	if r.host != nil {
+		base.Host, err = r.host(host)
+		if err != nil {
+			return nil, err
 		}
 	}
 
+	base.Scheme = "https"
+	if r.plainHTTP || strings.HasPrefix(base.Host, "localhost:") {
+		base.Scheme = "http"
+	}
+
 	if r.credentials != nil {
 		username, secret, err = r.credentials(base.Host)
 		if err != nil {