Add registry auth config, and use docker resolver in containerd.

Signed-off-by: Lantao Liu <lantaol@google.com>

pkg/config/config.go

@@ -61,16 +61,32 @@ type CniConfig struct {
 // Mirror contains the config related to the registry mirror
 type Mirror struct {
 	// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
-	// one by one until a working one is found.
+	// one by one until a working one is found. The endpoint must be a valid url
+	// with host specified.
 	Endpoints []string `toml:"endpoint" json:"endpoint"`
-	// TODO (Abhi) We might need to add auth per namespace. Looks like
-	// image auth information is passed by kube itself.
+}
+
+// AuthConfig contains the config related to authentication to a specific registry
+type AuthConfig struct {
+	// Username is the username to login the registry.
+	Username string `toml:"username" json:"username"`
+	// Password is the password to login the registry.
+	Password string `toml:"password" json:"password"`
+	// Auth is a base64 encoded string from the concatenation of the username,
+	// a colon, and the password.
+	Auth string `toml:"auth" json:"auth"`
+	// IdentityToken is used to authenticate the user and get
+	// an access token for the registry.
+	IdentityToken string `toml:"identitytoken" json:"identitytoken"`
 }
 
 // Registry is registry settings configured
 type Registry struct {
 	// Mirrors are namespace to mirror mapping for all namespaces.
 	Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"`
+	// Auths are registry endpoint to auth config mapping. The registry endpoint must
+	// be a valid url with host specified.
+	Auths map[string]AuthConfig `toml:"auths" json:"auths"`
 }
 
 // PluginConfig contains toml config related to CRI plugin,
@@ -81,7 +97,7 @@ type PluginConfig struct {
 	// CniConfig contains config related to cni
 	CniConfig `toml:"cni" json:"cni"`
 	// Registry contains config related to the registry
-	Registry `toml:"registry" json:"registry"`
+	Registry Registry `toml:"registry" json:"registry"`
 	// StreamServerAddress is the ip address streaming server is listening on.
 	StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
 	// StreamServerPort is the port streaming server is listening on.

pkg/server/helpers.go

@@ -444,3 +444,13 @@ func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtim
 	r.Root = runtimeOpts.RuntimeRoot
 	return r, nil
 }
+
+// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
+func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
+	return &runtime.AuthConfig{
+		Username:      a.Username,
+		Password:      a.Password,
+		Auth:          a.Auth,
+		IdentityToken: a.IdentityToken,
+	}
+}

pkg/server/image_pull.go

@@ -19,18 +19,21 @@ package server
 import (
 	"encoding/base64"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
-	containerdresolver "github.com/containerd/cri/pkg/containerd/resolver"
 	imagestore "github.com/containerd/cri/pkg/store/image"
 	"github.com/containerd/cri/pkg/util"
 )
@@ -87,12 +90,7 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
 	if ref != imageRef {
 		logrus.Debugf("PullImage using normalized image ref: %q", ref)
 	}
-	resolver := containerdresolver.NewResolver(containerdresolver.Options{
-		Credentials: func(string) (string, string, error) { return ParseAuth(r.GetAuth()) },
-		Client:      http.DefaultClient,
-		Registry:    c.getResolverOptions(),
-	})
-	_, desc, err := resolver.Resolve(ctx, ref)
+	resolver, desc, err := c.getResolver(ctx, ref, c.credentials(r.GetAuth()))
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to resolve image %q", ref)
 	}
@@ -206,10 +204,59 @@ func (c *criService) createImageReference(ctx context.Context, name string, desc
 	return err
 }
 
-func (c *criService) getResolverOptions() map[string][]string {
-	options := make(map[string][]string)
-	for ns, mirror := range c.config.Mirrors {
-		options[ns] = append(options[ns], mirror.Endpoints...)
+// credentials returns a credential function for docker resolver to use.
+func (c *criService) credentials(auth *runtime.AuthConfig) func(string) (string, string, error) {
+	return func(host string) (string, string, error) {
+		if auth == nil {
+			// Get default auth from config.
+			for h, ac := range c.config.Registry.Auths {
+				u, err := url.Parse(h)
+				if err != nil {
+					return "", "", errors.Wrapf(err, "parse auth host %q", h)
+				}
+				if u.Host == host {
+					auth = toRuntimeAuthConfig(ac)
+					break
+				}
+			}
+		}
+		return ParseAuth(auth)
 	}
-	return options
+}
+
+// getResolver tries registry mirrors and the default registry, and returns the resolver and descriptor
+// from the first working registry.
+func (c *criService) getResolver(ctx context.Context, ref string, cred func(string) (string, string, error)) (remotes.Resolver, imagespec.Descriptor, error) {
+	refspec, err := reference.Parse(ref)
+	if err != nil {
+		return nil, imagespec.Descriptor{}, errors.Wrap(err, "parse image reference")
+	}
+	// Try mirrors in order first, and then try default host name.
+	for _, e := range c.config.Registry.Mirrors[refspec.Hostname()].Endpoints {
+		u, err := url.Parse(e)
+		if err != nil {
+			return nil, imagespec.Descriptor{}, errors.Wrapf(err, "parse registry endpoint %q", e)
+		}
+		resolver := docker.NewResolver(docker.ResolverOptions{
+			Credentials: cred,
+			Client:      http.DefaultClient,
+			Host:        func(string) (string, error) { return u.Host, nil },
+			// By default use "https".
+			PlainHTTP: u.Scheme == "http",
+		})
+		_, desc, err := resolver.Resolve(ctx, ref)
+		if err == nil {
+			return resolver, desc, nil
+		}
+		// Continue to try next endpoint
+	}
+	resolver := docker.NewResolver(docker.ResolverOptions{
+		Credentials: cred,
+		Client:      http.DefaultClient,
+	})
+	_, desc, err := resolver.Resolve(ctx, ref)
+	if err != nil {
+		return nil, imagespec.Descriptor{}, errors.Wrap(err, "no available registry endpoint")
+	}
+	return resolver, desc, nil
 }

pkg/server/image_pull_test.go

@@ -22,6 +22,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/cri/pkg/config"
 )
 
 func TestParseAuth(t *testing.T) {
@@ -72,3 +74,58 @@ func TestParseAuth(t *testing.T) {
 		assert.Equal(t, test.expectedSecret, s)
 	}
 }
+
+func TestCredentials(t *testing.T) {
+	c := newTestCRIService()
+	c.config.Registry.Auths = map[string]criconfig.AuthConfig{
+		"https://test1.io": {
+			Username: "username1",
+			Password: "password1",
+		},
+		"http://test2.io": {
+			Username: "username2",
+			Password: "password2",
+		},
+		"//test3.io": {
+			Username: "username3",
+			Password: "password3",
+		},
+	}
+	for desc, test := range map[string]struct {
+		auth             *runtime.AuthConfig
+		host             string
+		expectedUsername string
+		expectedPassword string
+	}{
+		"auth config from CRI should take precedence": {
+			auth: &runtime.AuthConfig{
+				Username: "username",
+				Password: "password",
+			},
+			host:             "test1.io",
+			expectedUsername: "username",
+			expectedPassword: "password",
+		},
+		"should support https host": {
+			host:             "test1.io",
+			expectedUsername: "username1",
+			expectedPassword: "password1",
+		},
+		"should support http host": {
+			host:             "test2.io",
+			expectedUsername: "username2",
+			expectedPassword: "password2",
+		},
+		"should support hostname only": {
+			host:             "test3.io",
+			expectedUsername: "username3",
+			expectedPassword: "password3",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		username, password, err := c.credentials(test.auth)(test.host)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expectedUsername, username)
+		assert.Equal(t, test.expectedPassword, password)
+	}
+}