Use containerd registry mirror library.

Signed-off-by: Lantao Liu <lantaol@google.com>

cri.go

@@ -170,6 +170,19 @@ func validateConfig(ctx context.Context, c *criconfig.Config) error {
 		}
 	}
 
+	// Validation for registry configurations.
+	if len(c.Registry.Auths) != 0 {
+		if c.Registry.Configs == nil {
+			c.Registry.Configs = make(map[string]criconfig.RegistryConfig)
+		}
+		for endpoint, auth := range c.Registry.Auths {
+			config := c.Registry.Configs[endpoint]
+			config.Auth = &auth
+			c.Registry.Configs[endpoint] = config
+		}
+		log.G(ctx).Warning("`auths` is deprecated, please use registry`configs` instead")
+	}
+
 	// Validation for stream_idle_timeout
 	if c.StreamIdleTimeout != "" {
 		if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil {

docs/registry.md

@@ -27,8 +27,8 @@ After modify this config, you need restart the `containerd` service.
 
 To configure the TLS settings for a specific registry, create/modify the `/etc/containerd/config.toml` as follows:
 ```toml
-[plugins.cri.registry.tls_configs]
+# The registry host has to be an FDQN or IP.
-  [plugins.cri.registry.tls_configs."my.custom.registry"]
+[plugins.cri.registry.configs."my.custom.registry".tls]
     ca_file   = "ca.pem"
     cert_file = "cert.pem"
     key_file  = "key.pem"
@@ -37,16 +37,15 @@ To configure the TLS settings for a specific registry, create/modify the `/etc/c
 In the config example shown above, TLS mutual authentication will be used for communications with the registry endpoint located at https://my.custom.registry.
 `ca_file` is file name of the certificate authority (CA) certificate used to authenticate the x509 certificate/key pair specified by the files respectively pointed to by `cert_file` and `key_file`.
 
-
 ## Configure Registry Credentials
 
 `cri` plugin also supports docker like registry credential config.
 
-To configure a credential for a specific registry endpoint, create/modify the
+To configure a credential for a specific registry, create/modify the
 `/etc/containerd/config.toml` as follows:
 ```toml
-[plugins.cri.registry.auths]
+# The registry host has to be an FDQN or IP.
-  [plugins.cri.registry.auths."https://gcr.io"]
+[plugins.cri.registry.configs."gcr.io".auth]
   username = ""
   password = ""
   auth = ""

pkg/config/config.go

@@ -95,6 +95,7 @@ type Mirror struct {
 	// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
 	// one by one until a working one is found. The endpoint must be a valid url
 	// with host specified.
+	// The scheme, host and path from the endpoint URL will be used.
 	Endpoints []string `toml:"endpoint" json:"endpoint"`
 }
 
@@ -123,12 +124,23 @@ type TLSConfig struct {
 type Registry struct {
 	// Mirrors are namespace to mirror mapping for all namespaces.
 	Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"`
+	// Configs are configs for each registry.
+	// The key is the FDQN or IP of the registry.
+	Configs map[string]RegistryConfig `toml:"configs" json:"configs"`
+
 	// Auths are registry endpoint to auth config mapping. The registry endpoint must
 	// be a valid url with host specified.
+	// DEPRECATED: Use Configs instead. Remove in containerd 1.4.
 	Auths map[string]AuthConfig `toml:"auths" json:"auths"`
-	// TLSConfigs are pairs of CA/Cert/Key which then are used when creating the transport
+}
+
+// RegistryConfig contains configuration used to communicate with the registry.
+type RegistryConfig struct {
+	// Auth contains information to authenticate to the registry.
+	Auth *AuthConfig `toml:"auth" json:"auth"`
+	// TLS is a pair of CA/Cert/Key which then are used when creating the transport
 	// that communicates with the registry.
-	TLSConfigs map[string]TLSConfig `toml:"tls_configs" json:"tlsConfigs"`
+	TLS *TLSConfig `toml:"tls" json:"tls"`
 }
 
 // PluginConfig contains toml config related to CRI plugin,

pkg/server/image_pull.go

@@ -31,16 +31,15 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/cri/pkg/util"
 	distribution "github.com/docker/distribution/reference"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 
-	"github.com/containerd/cri/pkg/config"
+	criconfig "github.com/containerd/cri/pkg/config"
 )
 
 // For image management:
@@ -95,7 +94,10 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
 	if ref != imageRef {
 		log.G(ctx).Debugf("PullImage using normalized image ref: %q", ref)
 	}
-	resolver, desc, err := c.getResolver(ctx, ref, c.credentials(r.GetAuth()))
+	resolver := docker.NewResolver(docker.ResolverOptions{
+		Hosts: c.registryHosts(r.GetAuth()),
+	})
+	_, desc, err := resolver.Resolve(ctx, ref)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to resolve image %q", ref)
 	}
@@ -148,10 +150,20 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
 }
 
 // ParseAuth parses AuthConfig and returns username and password/secret required by containerd.
-func ParseAuth(auth *runtime.AuthConfig) (string, string, error) {
+func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) {
 	if auth == nil {
 		return "", "", nil
 	}
+	if auth.ServerAddress != "" {
+		// Do not return the auth info when server address doesn't match.
+		u, err := url.Parse(auth.ServerAddress)
+		if err != nil {
+			return "", "", errors.Wrap(err, "parse server address")
+		}
+		if host != u.Host {
+			return "", "", nil
+		}
+	}
 	if auth.Username != "" {
 		return auth.Username, auth.Password, nil
 	}
@@ -235,28 +247,8 @@ func (c *criService) updateImage(ctx context.Context, r string) error {
 	return nil
 }
 
-// credentials returns a credential function for docker resolver to use.
-func (c *criService) credentials(auth *runtime.AuthConfig) func(string) (string, string, error) {
-	return func(host string) (string, string, error) {
-		if auth == nil {
-			// Get default auth from config.
-			for h, ac := range c.config.Registry.Auths {
-				u, err := url.Parse(h)
-				if err != nil {
-					return "", "", errors.Wrapf(err, "parse auth host %q", h)
-				}
-				if u.Host == host {
-					auth = toRuntimeAuthConfig(ac)
-					break
-				}
-			}
-		}
-		return ParseAuth(auth)
-	}
-}
-
 // getTLSConfig returns a TLSConfig configured with a CA/Cert/Key specified by registryTLSConfig
-func (c *criService) getTLSConfig(registryTLSConfig config.TLSConfig) (*tls.Config, error) {
+func (c *criService) getTLSConfig(registryTLSConfig criconfig.TLSConfig) (*tls.Config, error) {
 	cert, err := tls.LoadX509KeyPair(registryTLSConfig.CertFile, registryTLSConfig.KeyFile)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load cert file")
@@ -280,68 +272,79 @@ func (c *criService) getTLSConfig(registryTLSConfig config.TLSConfig) (*tls.Conf
 	return tlsConfig, nil
 }
 
-// getResolver tries registry mirrors and the default registry, and returns the resolver and descriptor
+// registryHosts is the registry hosts to be used by the resolver.
-// from the first working registry.
+func (c *criService) registryHosts(auth *runtime.AuthConfig) docker.RegistryHosts {
-func (c *criService) getResolver(ctx context.Context, ref string, cred func(string) (string, string, error)) (remotes.Resolver, imagespec.Descriptor, error) {
+	return func(host string) ([]docker.RegistryHost, error) {
-	refspec, err := reference.Parse(ref)
+		var registries []docker.RegistryHost
+
+		// Try mirrors in order, and then try the default registry if not tried.
+		endpoints, err := addDefaultEndpoint(
+			c.config.Registry.Mirrors[host].Endpoints, host)
 		if err != nil {
-		return nil, imagespec.Descriptor{}, errors.Wrap(err, "parse image reference")
+			return nil, errors.Wrapf(err, "add default endpoint")
+		}
+		for _, e := range endpoints {
+			u, err := url.Parse(e)
+			if err != nil {
+				return nil, errors.Wrapf(err, "parse registry endpoint %q from mirrors", e)
 			}
 
 			var (
 				transport = newTransport()
-		httpClient = &http.Client{Transport: transport}
+				client    = &http.Client{Transport: transport}
+				config    = c.config.Registry.Configs[u.Host]
 			)
 
-	// Try mirrors in order first, and then try default host name.
+			if u.Scheme != "https" && config.TLS != nil {
-	for _, e := range c.config.Registry.Mirrors[refspec.Hostname()].Endpoints {
+				return nil, errors.Errorf("tls provided for http endpoint %q", e)
-		u, err := url.Parse(e)
-		if err != nil {
-			return nil, imagespec.Descriptor{}, errors.Wrapf(err, "parse registry endpoint %q", e)
 			}
 
-		if registryTLSConfig, ok := c.config.Registry.TLSConfigs[u.Host]; ok {
+			if config.TLS != nil {
-			transport.TLSClientConfig, err = c.getTLSConfig(registryTLSConfig)
+				transport.TLSClientConfig, err = c.getTLSConfig(*config.TLS)
 				if err != nil {
-				return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get TLSConfig for registry %q", refspec.Hostname())
+					return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e)
 				}
 			}
 
-		resolver := docker.NewResolver(docker.ResolverOptions{
+			if auth == nil && config.Auth != nil {
-			Authorizer: docker.NewAuthorizer(httpClient, cred),
+				auth = toRuntimeAuthConfig(*config.Auth)
-			Client:     httpClient,
+			}
-			Host:       func(string) (string, error) { return u.Host, nil },
+
-			// By default use "https".
+			if u.Path == "" {
-			PlainHTTP: u.Scheme == "http",
+				u.Path = "/v2"
+			}
+
+			registries = append(registries, docker.RegistryHost{
+				Client: client,
+				Authorizer: docker.NewDockerAuthorizer(
+					docker.WithAuthClient(client),
+					docker.WithAuthCreds(func(host string) (string, string, error) {
+						return ParseAuth(auth, host)
+					})),
+				Host:         u.Host,
+				Scheme:       u.Scheme,
+				Path:         u.Path,
+				Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull,
 			})
-		_, desc, err := resolver.Resolve(ctx, ref)
-		if err == nil {
-			return resolver, desc, nil
 		}
-		log.G(ctx).WithError(err).Debugf("Tried registry mirror %q but failed", e)
+		return registries, nil
-		// Continue to try next endpoint
 	}
+}
 
-	hostname, err := docker.DefaultHost(refspec.Hostname())
+// addDefaultEndpoint add default registry endpoint if it does not
+// exist in the passed-in endpoint list.
+func addDefaultEndpoint(endpoints []string, host string) ([]string, error) {
+	defaultHost, err := docker.DefaultHost(host)
 	if err != nil {
-		return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get host for refspec %q", refspec.Hostname())
+		return nil, errors.Wrapf(err, "get default host")
 	}
-	if registryTLSConfig, ok := c.config.Registry.TLSConfigs[hostname]; ok {
+	// If the http endpoint is configured, do not try https.
-		transport.TLSClientConfig, err = c.getTLSConfig(registryTLSConfig)
+	if !util.InStringSlice(endpoints, "http://"+defaultHost) {
-		if err != nil {
+		if !util.InStringSlice(endpoints, "https://"+defaultHost) {
-			return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get TLSConfig for registry %q", refspec.Hostname())
+			return append(endpoints, "https://"+defaultHost), nil
 		}
 	}
-
+	return endpoints, nil
-	resolver := docker.NewResolver(docker.ResolverOptions{
-		Credentials: cred,
-		Client:      httpClient,
-	})
-	_, desc, err := resolver.Resolve(ctx, ref)
-	if err != nil {
-		return nil, imagespec.Descriptor{}, errors.Wrap(err, "no available registry endpoint")
-	}
-	return resolver, desc, nil
 }
 
 // newTransport returns a new HTTP transport used to pull image.

pkg/server/image_pull_test.go

@@ -22,8 +22,6 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-
-	criconfig "github.com/containerd/cri/pkg/config"
 )
 
 func TestParseAuth(t *testing.T) {
@@ -36,6 +34,7 @@ func TestParseAuth(t *testing.T) {
 	base64.StdEncoding.Encode(invalidAuth, []byte(testUser+"@"+testPasswd))
 	for desc, test := range map[string]struct {
 		auth           *runtime.AuthConfig
+		host           string
 		expectedUser   string
 		expectedSecret string
 		expectErr      bool
@@ -66,66 +65,92 @@ func TestParseAuth(t *testing.T) {
 			auth:      &runtime.AuthConfig{Auth: string(invalidAuth)},
 			expectErr: true,
 		},
+		"should return empty auth if server address doesn't match": {
+			auth: &runtime.AuthConfig{
+				Username:      testUser,
+				Password:      testPasswd,
+				ServerAddress: "https://registry-1.io",
+			},
+			host:           "registry-2.io",
+			expectedUser:   "",
+			expectedSecret: "",
+		},
+		"should return auth if server address matches": {
+			auth: &runtime.AuthConfig{
+				Username:      testUser,
+				Password:      testPasswd,
+				ServerAddress: "https://registry-1.io",
+			},
+			host:           "registry-1.io",
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+		"should return auth if server address is not specified": {
+			auth: &runtime.AuthConfig{
+				Username: testUser,
+				Password: testPasswd,
+			},
+			host:           "registry-1.io",
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
 	} {
 		t.Logf("TestCase %q", desc)
-		u, s, err := ParseAuth(test.auth)
+		u, s, err := ParseAuth(test.auth, test.host)
 		assert.Equal(t, test.expectErr, err != nil)
 		assert.Equal(t, test.expectedUser, u)
 		assert.Equal(t, test.expectedSecret, s)
 	}
 }
 
-func TestCredentials(t *testing.T) {
+func TestAddDefaultEndpoint(t *testing.T) {
-	c := newTestCRIService()
-	c.config.Registry.Auths = map[string]criconfig.AuthConfig{
-		"https://test1.io": {
-			Username: "username1",
-			Password: "password1",
-		},
-		"http://test2.io": {
-			Username: "username2",
-			Password: "password2",
-		},
-		"//test3.io": {
-			Username: "username3",
-			Password: "password3",
-		},
-	}
 	for desc, test := range map[string]struct {
-		auth             *runtime.AuthConfig
+		endpoints []string
 		host      string
-		expectedUsername string
+		expected  []string
-		expectedPassword string
 	}{
-		"auth config from CRI should take precedence": {
+		"default endpoint not in list": {
-			auth: &runtime.AuthConfig{
+			endpoints: []string{
-				Username: "username",
+				"https://registry-1.io",
-				Password: "password",
+				"https://registry-2.io",
 			},
-			host:             "test1.io",
+			host: "registry-3.io",
-			expectedUsername: "username",
+			expected: []string{
-			expectedPassword: "password",
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
 			},
-		"should support https host": {
-			host:             "test1.io",
-			expectedUsername: "username1",
-			expectedPassword: "password1",
 		},
-		"should support http host": {
+		"default endpoint in list with http": {
-			host:             "test2.io",
+			endpoints: []string{
-			expectedUsername: "username2",
+				"https://registry-1.io",
-			expectedPassword: "password2",
+				"https://registry-2.io",
+				"http://registry-3.io",
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"http://registry-3.io",
+			},
+		},
+		"default endpoint in list with https": {
+			endpoints: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
 			},
-		"should support hostname only": {
-			host:             "test3.io",
-			expectedUsername: "username3",
-			expectedPassword: "password3",
 		},
 	} {
 		t.Logf("TestCase %q", desc)
-		username, password, err := c.credentials(test.auth)(test.host)
+		got, err := addDefaultEndpoint(test.endpoints, test.host)
 		assert.NoError(t, err)
-		assert.Equal(t, test.expectedUsername, username)
+		assert.Equal(t, test.expected, got)
-		assert.Equal(t, test.expectedPassword, password)
 	}
 }