From 53e94c67537b3a33b497817619833c0eb234f6e9 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 6 Aug 2019 18:19:40 -0700 Subject: [PATCH] Use containerd registry mirror library. Signed-off-by: Lantao Liu --- cri.go | 13 +++ docs/registry.md | 19 ++--- pkg/config/config.go | 16 +++- pkg/server/image_pull.go | 153 +++++++++++++++++----------------- pkg/server/image_pull_test.go | 117 ++++++++++++++++---------- 5 files changed, 185 insertions(+), 133 deletions(-) diff --git a/cri.go b/cri.go index 71fe55ee5..e8bd5987b 100644 --- a/cri.go +++ b/cri.go @@ -170,6 +170,19 @@ func validateConfig(ctx context.Context, c *criconfig.Config) error { } } + // Validation for registry configurations. + if len(c.Registry.Auths) != 0 { + if c.Registry.Configs == nil { + c.Registry.Configs = make(map[string]criconfig.RegistryConfig) + } + for endpoint, auth := range c.Registry.Auths { + config := c.Registry.Configs[endpoint] + config.Auth = &auth + c.Registry.Configs[endpoint] = config + } + log.G(ctx).Warning("`auths` is deprecated, please use registry`configs` instead") + } + // Validation for stream_idle_timeout if c.StreamIdleTimeout != "" { if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil { diff --git a/docs/registry.md b/docs/registry.md index f73ce0824..46e5b297e 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -27,8 +27,8 @@ After modify this config, you need restart the `containerd` service. To configure the TLS settings for a specific registry, create/modify the `/etc/containerd/config.toml` as follows: ```toml -[plugins.cri.registry.tls_configs] - [plugins.cri.registry.tls_configs."my.custom.registry"] +# The registry host has to be an FDQN or IP. +[plugins.cri.registry.configs."my.custom.registry".tls] ca_file = "ca.pem" cert_file = "cert.pem" key_file = "key.pem" @@ -37,20 +37,19 @@ To configure the TLS settings for a specific registry, create/modify the `/etc/c In the config example shown above, TLS mutual authentication will be used for communications with the registry endpoint located at https://my.custom.registry. `ca_file` is file name of the certificate authority (CA) certificate used to authenticate the x509 certificate/key pair specified by the files respectively pointed to by `cert_file` and `key_file`. - ## Configure Registry Credentials `cri` plugin also supports docker like registry credential config. -To configure a credential for a specific registry endpoint, create/modify the +To configure a credential for a specific registry, create/modify the `/etc/containerd/config.toml` as follows: ```toml -[plugins.cri.registry.auths] - [plugins.cri.registry.auths."https://gcr.io"] - username = "" - password = "" - auth = "" - identitytoken = "" +# The registry host has to be an FDQN or IP. +[plugins.cri.registry.configs."gcr.io".auth] + username = "" + password = "" + auth = "" + identitytoken = "" ``` The meaning of each field is the same with the corresponding field in `.docker/config.json`. diff --git a/pkg/config/config.go b/pkg/config/config.go index dc39616e1..ad78b5d98 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -95,6 +95,7 @@ type Mirror struct { // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints // one by one until a working one is found. The endpoint must be a valid url // with host specified. + // The scheme, host and path from the endpoint URL will be used. Endpoints []string `toml:"endpoint" json:"endpoint"` } @@ -123,12 +124,23 @@ type TLSConfig struct { type Registry struct { // Mirrors are namespace to mirror mapping for all namespaces. Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"` + // Configs are configs for each registry. + // The key is the FDQN or IP of the registry. + Configs map[string]RegistryConfig `toml:"configs" json:"configs"` + // Auths are registry endpoint to auth config mapping. The registry endpoint must // be a valid url with host specified. + // DEPRECATED: Use Configs instead. Remove in containerd 1.4. Auths map[string]AuthConfig `toml:"auths" json:"auths"` - // TLSConfigs are pairs of CA/Cert/Key which then are used when creating the transport +} + +// RegistryConfig contains configuration used to communicate with the registry. +type RegistryConfig struct { + // Auth contains information to authenticate to the registry. + Auth *AuthConfig `toml:"auth" json:"auth"` + // TLS is a pair of CA/Cert/Key which then are used when creating the transport // that communicates with the registry. - TLSConfigs map[string]TLSConfig `toml:"tls_configs" json:"tlsConfigs"` + TLS *TLSConfig `toml:"tls" json:"tls"` } // PluginConfig contains toml config related to CRI plugin, diff --git a/pkg/server/image_pull.go b/pkg/server/image_pull.go index 09d23ebcb..4e68ed111 100644 --- a/pkg/server/image_pull.go +++ b/pkg/server/image_pull.go @@ -31,16 +31,15 @@ import ( "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" - "github.com/containerd/containerd/reference" - "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" + "github.com/containerd/cri/pkg/util" distribution "github.com/docker/distribution/reference" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/net/context" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - "github.com/containerd/cri/pkg/config" + criconfig "github.com/containerd/cri/pkg/config" ) // For image management: @@ -95,7 +94,10 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) if ref != imageRef { log.G(ctx).Debugf("PullImage using normalized image ref: %q", ref) } - resolver, desc, err := c.getResolver(ctx, ref, c.credentials(r.GetAuth())) + resolver := docker.NewResolver(docker.ResolverOptions{ + Hosts: c.registryHosts(r.GetAuth()), + }) + _, desc, err := resolver.Resolve(ctx, ref) if err != nil { return nil, errors.Wrapf(err, "failed to resolve image %q", ref) } @@ -148,10 +150,20 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) } // ParseAuth parses AuthConfig and returns username and password/secret required by containerd. -func ParseAuth(auth *runtime.AuthConfig) (string, string, error) { +func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) { if auth == nil { return "", "", nil } + if auth.ServerAddress != "" { + // Do not return the auth info when server address doesn't match. + u, err := url.Parse(auth.ServerAddress) + if err != nil { + return "", "", errors.Wrap(err, "parse server address") + } + if host != u.Host { + return "", "", nil + } + } if auth.Username != "" { return auth.Username, auth.Password, nil } @@ -235,28 +247,8 @@ func (c *criService) updateImage(ctx context.Context, r string) error { return nil } -// credentials returns a credential function for docker resolver to use. -func (c *criService) credentials(auth *runtime.AuthConfig) func(string) (string, string, error) { - return func(host string) (string, string, error) { - if auth == nil { - // Get default auth from config. - for h, ac := range c.config.Registry.Auths { - u, err := url.Parse(h) - if err != nil { - return "", "", errors.Wrapf(err, "parse auth host %q", h) - } - if u.Host == host { - auth = toRuntimeAuthConfig(ac) - break - } - } - } - return ParseAuth(auth) - } -} - // getTLSConfig returns a TLSConfig configured with a CA/Cert/Key specified by registryTLSConfig -func (c *criService) getTLSConfig(registryTLSConfig config.TLSConfig) (*tls.Config, error) { +func (c *criService) getTLSConfig(registryTLSConfig criconfig.TLSConfig) (*tls.Config, error) { cert, err := tls.LoadX509KeyPair(registryTLSConfig.CertFile, registryTLSConfig.KeyFile) if err != nil { return nil, errors.Wrap(err, "failed to load cert file") @@ -280,68 +272,79 @@ func (c *criService) getTLSConfig(registryTLSConfig config.TLSConfig) (*tls.Conf return tlsConfig, nil } -// getResolver tries registry mirrors and the default registry, and returns the resolver and descriptor -// from the first working registry. -func (c *criService) getResolver(ctx context.Context, ref string, cred func(string) (string, string, error)) (remotes.Resolver, imagespec.Descriptor, error) { - refspec, err := reference.Parse(ref) - if err != nil { - return nil, imagespec.Descriptor{}, errors.Wrap(err, "parse image reference") - } +// registryHosts is the registry hosts to be used by the resolver. +func (c *criService) registryHosts(auth *runtime.AuthConfig) docker.RegistryHosts { + return func(host string) ([]docker.RegistryHost, error) { + var registries []docker.RegistryHost - var ( - transport = newTransport() - httpClient = &http.Client{Transport: transport} - ) - - // Try mirrors in order first, and then try default host name. - for _, e := range c.config.Registry.Mirrors[refspec.Hostname()].Endpoints { - u, err := url.Parse(e) + // Try mirrors in order, and then try the default registry if not tried. + endpoints, err := addDefaultEndpoint( + c.config.Registry.Mirrors[host].Endpoints, host) if err != nil { - return nil, imagespec.Descriptor{}, errors.Wrapf(err, "parse registry endpoint %q", e) + return nil, errors.Wrapf(err, "add default endpoint") } - - if registryTLSConfig, ok := c.config.Registry.TLSConfigs[u.Host]; ok { - transport.TLSClientConfig, err = c.getTLSConfig(registryTLSConfig) + for _, e := range endpoints { + u, err := url.Parse(e) if err != nil { - return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get TLSConfig for registry %q", refspec.Hostname()) + return nil, errors.Wrapf(err, "parse registry endpoint %q from mirrors", e) } - } - resolver := docker.NewResolver(docker.ResolverOptions{ - Authorizer: docker.NewAuthorizer(httpClient, cred), - Client: httpClient, - Host: func(string) (string, error) { return u.Host, nil }, - // By default use "https". - PlainHTTP: u.Scheme == "http", - }) - _, desc, err := resolver.Resolve(ctx, ref) - if err == nil { - return resolver, desc, nil + var ( + transport = newTransport() + client = &http.Client{Transport: transport} + config = c.config.Registry.Configs[u.Host] + ) + + if u.Scheme != "https" && config.TLS != nil { + return nil, errors.Errorf("tls provided for http endpoint %q", e) + } + + if config.TLS != nil { + transport.TLSClientConfig, err = c.getTLSConfig(*config.TLS) + if err != nil { + return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e) + } + } + + if auth == nil && config.Auth != nil { + auth = toRuntimeAuthConfig(*config.Auth) + } + + if u.Path == "" { + u.Path = "/v2" + } + + registries = append(registries, docker.RegistryHost{ + Client: client, + Authorizer: docker.NewDockerAuthorizer( + docker.WithAuthClient(client), + docker.WithAuthCreds(func(host string) (string, string, error) { + return ParseAuth(auth, host) + })), + Host: u.Host, + Scheme: u.Scheme, + Path: u.Path, + Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull, + }) } - log.G(ctx).WithError(err).Debugf("Tried registry mirror %q but failed", e) - // Continue to try next endpoint + return registries, nil } +} - hostname, err := docker.DefaultHost(refspec.Hostname()) +// addDefaultEndpoint add default registry endpoint if it does not +// exist in the passed-in endpoint list. +func addDefaultEndpoint(endpoints []string, host string) ([]string, error) { + defaultHost, err := docker.DefaultHost(host) if err != nil { - return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get host for refspec %q", refspec.Hostname()) + return nil, errors.Wrapf(err, "get default host") } - if registryTLSConfig, ok := c.config.Registry.TLSConfigs[hostname]; ok { - transport.TLSClientConfig, err = c.getTLSConfig(registryTLSConfig) - if err != nil { - return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get TLSConfig for registry %q", refspec.Hostname()) + // If the http endpoint is configured, do not try https. + if !util.InStringSlice(endpoints, "http://"+defaultHost) { + if !util.InStringSlice(endpoints, "https://"+defaultHost) { + return append(endpoints, "https://"+defaultHost), nil } } - - resolver := docker.NewResolver(docker.ResolverOptions{ - Credentials: cred, - Client: httpClient, - }) - _, desc, err := resolver.Resolve(ctx, ref) - if err != nil { - return nil, imagespec.Descriptor{}, errors.Wrap(err, "no available registry endpoint") - } - return resolver, desc, nil + return endpoints, nil } // newTransport returns a new HTTP transport used to pull image. diff --git a/pkg/server/image_pull_test.go b/pkg/server/image_pull_test.go index 83a7566f5..36fba0ce4 100644 --- a/pkg/server/image_pull_test.go +++ b/pkg/server/image_pull_test.go @@ -22,8 +22,6 @@ import ( "github.com/stretchr/testify/assert" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - - criconfig "github.com/containerd/cri/pkg/config" ) func TestParseAuth(t *testing.T) { @@ -36,6 +34,7 @@ func TestParseAuth(t *testing.T) { base64.StdEncoding.Encode(invalidAuth, []byte(testUser+"@"+testPasswd)) for desc, test := range map[string]struct { auth *runtime.AuthConfig + host string expectedUser string expectedSecret string expectErr bool @@ -66,66 +65,92 @@ func TestParseAuth(t *testing.T) { auth: &runtime.AuthConfig{Auth: string(invalidAuth)}, expectErr: true, }, + "should return empty auth if server address doesn't match": { + auth: &runtime.AuthConfig{ + Username: testUser, + Password: testPasswd, + ServerAddress: "https://registry-1.io", + }, + host: "registry-2.io", + expectedUser: "", + expectedSecret: "", + }, + "should return auth if server address matches": { + auth: &runtime.AuthConfig{ + Username: testUser, + Password: testPasswd, + ServerAddress: "https://registry-1.io", + }, + host: "registry-1.io", + expectedUser: testUser, + expectedSecret: testPasswd, + }, + "should return auth if server address is not specified": { + auth: &runtime.AuthConfig{ + Username: testUser, + Password: testPasswd, + }, + host: "registry-1.io", + expectedUser: testUser, + expectedSecret: testPasswd, + }, } { t.Logf("TestCase %q", desc) - u, s, err := ParseAuth(test.auth) + u, s, err := ParseAuth(test.auth, test.host) assert.Equal(t, test.expectErr, err != nil) assert.Equal(t, test.expectedUser, u) assert.Equal(t, test.expectedSecret, s) } } -func TestCredentials(t *testing.T) { - c := newTestCRIService() - c.config.Registry.Auths = map[string]criconfig.AuthConfig{ - "https://test1.io": { - Username: "username1", - Password: "password1", - }, - "http://test2.io": { - Username: "username2", - Password: "password2", - }, - "//test3.io": { - Username: "username3", - Password: "password3", - }, - } +func TestAddDefaultEndpoint(t *testing.T) { for desc, test := range map[string]struct { - auth *runtime.AuthConfig - host string - expectedUsername string - expectedPassword string + endpoints []string + host string + expected []string }{ - "auth config from CRI should take precedence": { - auth: &runtime.AuthConfig{ - Username: "username", - Password: "password", + "default endpoint not in list": { + endpoints: []string{ + "https://registry-1.io", + "https://registry-2.io", + }, + host: "registry-3.io", + expected: []string{ + "https://registry-1.io", + "https://registry-2.io", + "https://registry-3.io", }, - host: "test1.io", - expectedUsername: "username", - expectedPassword: "password", }, - "should support https host": { - host: "test1.io", - expectedUsername: "username1", - expectedPassword: "password1", + "default endpoint in list with http": { + endpoints: []string{ + "https://registry-1.io", + "https://registry-2.io", + "http://registry-3.io", + }, + host: "registry-3.io", + expected: []string{ + "https://registry-1.io", + "https://registry-2.io", + "http://registry-3.io", + }, }, - "should support http host": { - host: "test2.io", - expectedUsername: "username2", - expectedPassword: "password2", - }, - "should support hostname only": { - host: "test3.io", - expectedUsername: "username3", - expectedPassword: "password3", + "default endpoint in list with https": { + endpoints: []string{ + "https://registry-1.io", + "https://registry-2.io", + "https://registry-3.io", + }, + host: "registry-3.io", + expected: []string{ + "https://registry-1.io", + "https://registry-2.io", + "https://registry-3.io", + }, }, } { t.Logf("TestCase %q", desc) - username, password, err := c.credentials(test.auth)(test.host) + got, err := addDefaultEndpoint(test.endpoints, test.host) assert.NoError(t, err) - assert.Equal(t, test.expectedUsername, username) - assert.Equal(t, test.expectedPassword, password) + assert.Equal(t, test.expected, got) } }