From 952e53bf58a9e040edf8123357b0d31b39aad3c5 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 9 Jul 2018 00:58:00 -0700 Subject: [PATCH] Add registry auth config, and use docker resolver in containerd. Signed-off-by: Lantao Liu --- docs/installation.md | 1 - docs/registry.md | 25 ++++++- hack/verify-lint.sh | 2 +- pkg/config/config.go | 24 +++++-- pkg/server/helpers.go | 10 +++ pkg/server/image_pull.go | 71 +++++++++++++++---- pkg/server/image_pull_test.go | 57 +++++++++++++++ .../containerd/remotes/docker/resolver.go | 36 +++++++--- 8 files changed, 198 insertions(+), 28 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index f0e9d5a80..c3cc1bfee 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,4 +1,3 @@ - # Install Containerd with Release Tarball This document provides the steps to install `containerd` and its dependencies with the release tarball, and bring up a Kubernetes cluster using kubeadm. diff --git a/docs/registry.md b/docs/registry.md index d3dba4917..5b084128a 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -1,6 +1,7 @@ # Configure Image Registry This document describes the method to configure the image registry for `containerd` for use with the `cri` plugin. +## Configure Registry Endpoint With containerd, `docker.io` is the default image registry. You can also set up other image registries similar to docker. To configure image registries create/modify the `/etc/containerd/config.toml` as follows: @@ -19,4 +20,26 @@ The default configuration can be generated by `containerd config default > /etc/ The endpoint is a list that can contain multiple image registry URLs split by commas. When pulling an image from a registry, containerd will try these endpoint URLs one by one, and use the first working one. -After modify the config file, you need restart the `containerd` service. +After modify this config, you need restart the `containerd` service. + +## Configure Registry Credentials + +`cri` plugin also supports docker like registry credential config. + +To configure a credential for a specific registry endpoint, create/modify the +`/etc/containerd/config.toml` as follows: +```toml +[plugins.cri.registry.auths] + [plugins.cri.registry.auths."https://gcr.io"] + username = "" + password = "" + auth = "" + identitytoken = "" +``` +The meaning of each field is the same with the corresponding field in `.docker/config.json`. + +Please note that auth config passed by CRI takes precedence over this config. +The registry credential in this config will only be used when auth config is +not specified by Kubernetes via CRI. + +After modify this config, you need restart the `containerd` service. diff --git a/hack/verify-lint.sh b/hack/verify-lint.sh index 32731d55c..36ccec1c4 100755 --- a/hack/verify-lint.sh +++ b/hack/verify-lint.sh @@ -20,7 +20,7 @@ set -o pipefail for d in $(find . -type d -a \( -iwholename './pkg*' -o -iwholename './cmd*' \) -not -iwholename './pkg/api*'); do echo for directory ${d} ... gometalinter \ - --exclude='error return value not checked.*(Close|Log|Print).*\(errcheck\)$' \ + --exclude='error return value not checked.*(Close|Log|Print|Fprint).*\(errcheck\)$' \ --exclude='.*_test\.go:.*error return value not checked.*\(errcheck\)$' \ --exclude='duplicate of.*_test.go.*\(dupl\)$' \ --exclude='.*/mock_.*\.go:.*\(golint\)$' \ diff --git a/pkg/config/config.go b/pkg/config/config.go index 8ec2be834..e6cefbd4b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -61,16 +61,32 @@ type CniConfig struct { // Mirror contains the config related to the registry mirror type Mirror struct { // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints - // one by one until a working one is found. + // one by one until a working one is found. The endpoint must be a valid url + // with host specified. Endpoints []string `toml:"endpoint" json:"endpoint"` - // TODO (Abhi) We might need to add auth per namespace. Looks like - // image auth information is passed by kube itself. +} + +// AuthConfig contains the config related to authentication to a specific registry +type AuthConfig struct { + // Username is the username to login the registry. + Username string `toml:"username" json:"username"` + // Password is the password to login the registry. + Password string `toml:"password" json:"password"` + // Auth is a base64 encoded string from the concatenation of the username, + // a colon, and the password. + Auth string `toml:"auth" json:"auth"` + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `toml:"identitytoken" json:"identitytoken"` } // Registry is registry settings configured type Registry struct { // Mirrors are namespace to mirror mapping for all namespaces. Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"` + // Auths are registry endpoint to auth config mapping. The registry endpoint must + // be a valid url with host specified. + Auths map[string]AuthConfig `toml:"auths" json:"auths"` } // PluginConfig contains toml config related to CRI plugin, @@ -81,7 +97,7 @@ type PluginConfig struct { // CniConfig contains config related to cni CniConfig `toml:"cni" json:"cni"` // Registry contains config related to the registry - Registry `toml:"registry" json:"registry"` + Registry Registry `toml:"registry" json:"registry"` // StreamServerAddress is the ip address streaming server is listening on. StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"` // StreamServerPort is the port streaming server is listening on. diff --git a/pkg/server/helpers.go b/pkg/server/helpers.go index 5bfdb8205..29f2f9ba4 100644 --- a/pkg/server/helpers.go +++ b/pkg/server/helpers.go @@ -444,3 +444,13 @@ func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtim r.Root = runtimeOpts.RuntimeRoot return r, nil } + +// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config. +func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig { + return &runtime.AuthConfig{ + Username: a.Username, + Password: a.Password, + Auth: a.Auth, + IdentityToken: a.IdentityToken, + } +} diff --git a/pkg/server/image_pull.go b/pkg/server/image_pull.go index 67acddf0f..a25d8285e 100644 --- a/pkg/server/image_pull.go +++ b/pkg/server/image_pull.go @@ -19,18 +19,21 @@ package server import ( "encoding/base64" "net/http" + "net/url" "strings" "github.com/containerd/containerd" "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" + "github.com/containerd/containerd/reference" + "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/docker" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" - containerdresolver "github.com/containerd/cri/pkg/containerd/resolver" imagestore "github.com/containerd/cri/pkg/store/image" "github.com/containerd/cri/pkg/util" ) @@ -87,12 +90,7 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) if ref != imageRef { logrus.Debugf("PullImage using normalized image ref: %q", ref) } - resolver := containerdresolver.NewResolver(containerdresolver.Options{ - Credentials: func(string) (string, string, error) { return ParseAuth(r.GetAuth()) }, - Client: http.DefaultClient, - Registry: c.getResolverOptions(), - }) - _, desc, err := resolver.Resolve(ctx, ref) + resolver, desc, err := c.getResolver(ctx, ref, c.credentials(r.GetAuth())) if err != nil { return nil, errors.Wrapf(err, "failed to resolve image %q", ref) } @@ -206,10 +204,59 @@ func (c *criService) createImageReference(ctx context.Context, name string, desc return err } -func (c *criService) getResolverOptions() map[string][]string { - options := make(map[string][]string) - for ns, mirror := range c.config.Mirrors { - options[ns] = append(options[ns], mirror.Endpoints...) +// credentials returns a credential function for docker resolver to use. +func (c *criService) credentials(auth *runtime.AuthConfig) func(string) (string, string, error) { + return func(host string) (string, string, error) { + if auth == nil { + // Get default auth from config. + for h, ac := range c.config.Registry.Auths { + u, err := url.Parse(h) + if err != nil { + return "", "", errors.Wrapf(err, "parse auth host %q", h) + } + if u.Host == host { + auth = toRuntimeAuthConfig(ac) + break + } + } + } + return ParseAuth(auth) } - return options +} + +// getResolver tries registry mirrors and the default registry, and returns the resolver and descriptor +// from the first working registry. +func (c *criService) getResolver(ctx context.Context, ref string, cred func(string) (string, string, error)) (remotes.Resolver, imagespec.Descriptor, error) { + refspec, err := reference.Parse(ref) + if err != nil { + return nil, imagespec.Descriptor{}, errors.Wrap(err, "parse image reference") + } + // Try mirrors in order first, and then try default host name. + for _, e := range c.config.Registry.Mirrors[refspec.Hostname()].Endpoints { + u, err := url.Parse(e) + if err != nil { + return nil, imagespec.Descriptor{}, errors.Wrapf(err, "parse registry endpoint %q", e) + } + resolver := docker.NewResolver(docker.ResolverOptions{ + Credentials: cred, + Client: http.DefaultClient, + Host: func(string) (string, error) { return u.Host, nil }, + // By default use "https". + PlainHTTP: u.Scheme == "http", + }) + _, desc, err := resolver.Resolve(ctx, ref) + if err == nil { + return resolver, desc, nil + } + // Continue to try next endpoint + } + resolver := docker.NewResolver(docker.ResolverOptions{ + Credentials: cred, + Client: http.DefaultClient, + }) + _, desc, err := resolver.Resolve(ctx, ref) + if err != nil { + return nil, imagespec.Descriptor{}, errors.Wrap(err, "no available registry endpoint") + } + return resolver, desc, nil } diff --git a/pkg/server/image_pull_test.go b/pkg/server/image_pull_test.go index 1fa9d7703..83b591cf3 100644 --- a/pkg/server/image_pull_test.go +++ b/pkg/server/image_pull_test.go @@ -22,6 +22,8 @@ import ( "github.com/stretchr/testify/assert" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + + criconfig "github.com/containerd/cri/pkg/config" ) func TestParseAuth(t *testing.T) { @@ -72,3 +74,58 @@ func TestParseAuth(t *testing.T) { assert.Equal(t, test.expectedSecret, s) } } + +func TestCredentials(t *testing.T) { + c := newTestCRIService() + c.config.Registry.Auths = map[string]criconfig.AuthConfig{ + "https://test1.io": { + Username: "username1", + Password: "password1", + }, + "http://test2.io": { + Username: "username2", + Password: "password2", + }, + "//test3.io": { + Username: "username3", + Password: "password3", + }, + } + for desc, test := range map[string]struct { + auth *runtime.AuthConfig + host string + expectedUsername string + expectedPassword string + }{ + "auth config from CRI should take precedence": { + auth: &runtime.AuthConfig{ + Username: "username", + Password: "password", + }, + host: "test1.io", + expectedUsername: "username", + expectedPassword: "password", + }, + "should support https host": { + host: "test1.io", + expectedUsername: "username1", + expectedPassword: "password1", + }, + "should support http host": { + host: "test2.io", + expectedUsername: "username2", + expectedPassword: "password2", + }, + "should support hostname only": { + host: "test3.io", + expectedUsername: "username3", + expectedPassword: "password3", + }, + } { + t.Logf("TestCase %q", desc) + username, password, err := c.credentials(test.auth)(test.host) + assert.NoError(t, err) + assert.Equal(t, test.expectedUsername, username) + assert.Equal(t, test.expectedPassword, password) + } +} diff --git a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index eca02d532..f0a677c83 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -53,6 +53,7 @@ var ( type dockerResolver struct { credentials func(string) (string, string, error) + host func(string) (string, error) plainHTTP bool client *http.Client tracker StatusTracker @@ -65,6 +66,9 @@ type ResolverOptions struct { // is interpretted as a long lived token. Credentials func(string) (string, string, error) + // Host provides the hostname given a namespace. + Host func(string) (string, error) + // PlainHTTP specifies to use plain http and not https PlainHTTP bool @@ -77,14 +81,27 @@ type ResolverOptions struct { Tracker StatusTracker } +// DefaultHost is the default host function. +func DefaultHost(ns string) (string, error) { + if ns == "docker.io" { + return "registry-1.docker.io", nil + } + return ns, nil +} + // NewResolver returns a new resolver to a Docker registry func NewResolver(options ResolverOptions) remotes.Resolver { tracker := options.Tracker if tracker == nil { tracker = NewInMemoryTracker() } + host := options.Host + if host == nil { + host = DefaultHost + } return &dockerResolver{ credentials: options.Credentials, + host: host, plainHTTP: options.PlainHTTP, client: options.Client, tracker: tracker, @@ -270,18 +287,19 @@ func (r *dockerResolver) base(refspec reference.Spec) (*dockerBase, error) { ) host := refspec.Hostname() - base.Scheme = "https" - - if host == "docker.io" { - base.Host = "registry-1.docker.io" - } else { - base.Host = host - - if r.plainHTTP || strings.HasPrefix(host, "localhost:") { - base.Scheme = "http" + base.Host = host + if r.host != nil { + base.Host, err = r.host(host) + if err != nil { + return nil, err } } + base.Scheme = "https" + if r.plainHTTP || strings.HasPrefix(base.Host, "localhost:") { + base.Scheme = "http" + } + if r.credentials != nil { username, secret, err = r.credentials(base.Host) if err != nil {