github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Use containerd registry mirror library.

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu 2019-08-06 18:19:40 -07:00
parent 27de1a5862
commit 53e94c6753
5 changed files with 185 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cri.go
View File

@ -170,6 +170,19 @@ func validateConfig(ctx context.Context, c *criconfig.Config) error {
}
}
// Validation for registry configurations.
if len(c.Registry.Auths) != 0 {
if c.Registry.Configs == nil {
c.Registry.Configs = make(map[string]criconfig.RegistryConfig)
}
for endpoint, auth := range c.Registry.Auths {
config := c.Registry.Configs[endpoint]
config.Auth = &auth
c.Registry.Configs[endpoint] = config
}
log.G(ctx).Warning("`auths` is deprecated, please use registry`configs` instead")
}
// Validation for stream_idle_timeout
if c.StreamIdleTimeout != "" {
if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil {

11
docs/registry.md
View File

@ -27,8 +27,8 @@ After modify this config, you need restart the `containerd` service.
To configure the TLS settings for a specific registry, create/modify the `/etc/containerd/config.toml` as follows:
```toml
[plugins.cri.registry.tls_configs]
[plugins.cri.registry.tls_configs."my.custom.registry"]
# The registry host has to be an FDQN or IP.
[plugins.cri.registry.configs."my.custom.registry".tls]
ca_file = "ca.pem"
cert_file = "cert.pem"
key_file = "key.pem"
@ -37,16 +37,15 @@ To configure the TLS settings for a specific registry, create/modify the `/etc/c
In the config example shown above, TLS mutual authentication will be used for communications with the registry endpoint located at https://my.custom.registry.
`ca_file` is file name of the certificate authority (CA) certificate used to authenticate the x509 certificate/key pair specified by the files respectively pointed to by `cert_file` and `key_file`.
## Configure Registry Credentials
`cri` plugin also supports docker like registry credential config.
To configure a credential for a specific registry endpoint, create/modify the
To configure a credential for a specific registry, create/modify the
`/etc/containerd/config.toml` as follows:
```toml
[plugins.cri.registry.auths]
[plugins.cri.registry.auths."https://gcr.io"]
# The registry host has to be an FDQN or IP.
[plugins.cri.registry.configs."gcr.io".auth]
username = ""
password = ""
auth = ""

16
pkg/config/config.go
View File

@ -95,6 +95,7 @@ type Mirror struct {
// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
// one by one until a working one is found. The endpoint must be a valid url
// with host specified.
// The scheme, host and path from the endpoint URL will be used.
Endpoints []string `toml:"endpoint" json:"endpoint"`
}
@ -123,12 +124,23 @@ type TLSConfig struct {
type Registry struct {
// Mirrors are namespace to mirror mapping for all namespaces.
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"`
// Configs are configs for each registry.
// The key is the FDQN or IP of the registry.
Configs map[string]RegistryConfig `toml:"configs" json:"configs"`
// Auths are registry endpoint to auth config mapping. The registry endpoint must
// be a valid url with host specified.
// DEPRECATED: Use Configs instead. Remove in containerd 1.4.
Auths map[string]AuthConfig `toml:"auths" json:"auths"`
// TLSConfigs are pairs of CA/Cert/Key which then are used when creating the transport
}
// RegistryConfig contains configuration used to communicate with the registry.
type RegistryConfig struct {
// Auth contains information to authenticate to the registry.
Auth *AuthConfig `toml:"auth" json:"auth"`
// TLS is a pair of CA/Cert/Key which then are used when creating the transport
// that communicates with the registry.
TLSConfigs map[string]TLSConfig `toml:"tls_configs" json:"tlsConfigs"`
TLS *TLSConfig `toml:"tls" json:"tls"`
}
// PluginConfig contains toml config related to CRI plugin,

137
pkg/server/image_pull.go
View File

@ -31,16 +31,15 @@ import (
"github.com/containerd/containerd/errdefs"
containerdimages "github.com/containerd/containerd/images"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/cri/pkg/util"
distribution "github.com/docker/distribution/reference"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/net/context"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"github.com/containerd/cri/pkg/config"
criconfig "github.com/containerd/cri/pkg/config"
)
// For image management:
@ -95,7 +94,10 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
if ref != imageRef {
log.G(ctx).Debugf("PullImage using normalized image ref: %q", ref)
}
resolver, desc, err := c.getResolver(ctx, ref, c.credentials(r.GetAuth()))
resolver := docker.NewResolver(docker.ResolverOptions{
Hosts: c.registryHosts(r.GetAuth()),
})
_, desc, err := resolver.Resolve(ctx, ref)
if err != nil {
return nil, errors.Wrapf(err, "failed to resolve image %q", ref)
}
@ -148,10 +150,20 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
}
// ParseAuth parses AuthConfig and returns username and password/secret required by containerd.
func ParseAuth(auth *runtime.AuthConfig) (string, string, error) {
func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) {
if auth == nil {
return "", "", nil
}
if auth.ServerAddress != "" {
// Do not return the auth info when server address doesn't match.
u, err := url.Parse(auth.ServerAddress)
if err != nil {
return "", "", errors.Wrap(err, "parse server address")
}
if host != u.Host {
return "", "", nil
}
}
if auth.Username != "" {
return auth.Username, auth.Password, nil
}
@ -235,28 +247,8 @@ func (c *criService) updateImage(ctx context.Context, r string) error {
return nil
}
// credentials returns a credential function for docker resolver to use.
func (c *criService) credentials(auth *runtime.AuthConfig) func(string) (string, string, error) {
return func(host string) (string, string, error) {
if auth == nil {
// Get default auth from config.
for h, ac := range c.config.Registry.Auths {
u, err := url.Parse(h)
if err != nil {
return "", "", errors.Wrapf(err, "parse auth host %q", h)
}
if u.Host == host {
auth = toRuntimeAuthConfig(ac)
break
}
}
}
return ParseAuth(auth)
}
}
// getTLSConfig returns a TLSConfig configured with a CA/Cert/Key specified by registryTLSConfig
func (c *criService) getTLSConfig(registryTLSConfig config.TLSConfig) (*tls.Config, error) {
func (c *criService) getTLSConfig(registryTLSConfig criconfig.TLSConfig) (*tls.Config, error) {
cert, err := tls.LoadX509KeyPair(registryTLSConfig.CertFile, registryTLSConfig.KeyFile)
if err != nil {
return nil, errors.Wrap(err, "failed to load cert file")
@ -280,68 +272,79 @@ func (c *criService) getTLSConfig(registryTLSConfig config.TLSConfig) (*tls.Conf
return tlsConfig, nil
}
// getResolver tries registry mirrors and the default registry, and returns the resolver and descriptor
// from the first working registry.
func (c *criService) getResolver(ctx context.Context, ref string, cred func(string) (string, string, error)) (remotes.Resolver, imagespec.Descriptor, error) {
refspec, err := reference.Parse(ref)
// registryHosts is the registry hosts to be used by the resolver.
func (c *criService) registryHosts(auth *runtime.AuthConfig) docker.RegistryHosts {
return func(host string) ([]docker.RegistryHost, error) {
var registries []docker.RegistryHost
// Try mirrors in order, and then try the default registry if not tried.
endpoints, err := addDefaultEndpoint(
c.config.Registry.Mirrors[host].Endpoints, host)
if err != nil {
return nil, imagespec.Descriptor{}, errors.Wrap(err, "parse image reference")
return nil, errors.Wrapf(err, "add default endpoint")
}
for _, e := range endpoints {
u, err := url.Parse(e)
if err != nil {
return nil, errors.Wrapf(err, "parse registry endpoint %q from mirrors", e)
}
var (
transport = newTransport()
httpClient = &http.Client{Transport: transport}
client = &http.Client{Transport: transport}
config = c.config.Registry.Configs[u.Host]
)
// Try mirrors in order first, and then try default host name.
for _, e := range c.config.Registry.Mirrors[refspec.Hostname()].Endpoints {
u, err := url.Parse(e)
if err != nil {
return nil, imagespec.Descriptor{}, errors.Wrapf(err, "parse registry endpoint %q", e)
if u.Scheme != "https" && config.TLS != nil {
return nil, errors.Errorf("tls provided for http endpoint %q", e)
}
if registryTLSConfig, ok := c.config.Registry.TLSConfigs[u.Host]; ok {
transport.TLSClientConfig, err = c.getTLSConfig(registryTLSConfig)
if config.TLS != nil {
transport.TLSClientConfig, err = c.getTLSConfig(*config.TLS)
if err != nil {
return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get TLSConfig for registry %q", refspec.Hostname())
return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e)
}
}
resolver := docker.NewResolver(docker.ResolverOptions{
Authorizer: docker.NewAuthorizer(httpClient, cred),
Client: httpClient,
Host: func(string) (string, error) { return u.Host, nil },
// By default use "https".
PlainHTTP: u.Scheme == "http",
if auth == nil && config.Auth != nil {
auth = toRuntimeAuthConfig(*config.Auth)
}
if u.Path == "" {
u.Path = "/v2"
}
registries = append(registries, docker.RegistryHost{
Client: client,
Authorizer: docker.NewDockerAuthorizer(
docker.WithAuthClient(client),
docker.WithAuthCreds(func(host string) (string, string, error) {
return ParseAuth(auth, host)
})),
Host: u.Host,
Scheme: u.Scheme,
Path: u.Path,
Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull,
})
_, desc, err := resolver.Resolve(ctx, ref)
if err == nil {
return resolver, desc, nil
}
log.G(ctx).WithError(err).Debugf("Tried registry mirror %q but failed", e)
// Continue to try next endpoint
return registries, nil
}
}
hostname, err := docker.DefaultHost(refspec.Hostname())
// addDefaultEndpoint add default registry endpoint if it does not
// exist in the passed-in endpoint list.
func addDefaultEndpoint(endpoints []string, host string) ([]string, error) {
defaultHost, err := docker.DefaultHost(host)
if err != nil {
return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get host for refspec %q", refspec.Hostname())
return nil, errors.Wrapf(err, "get default host")
}
if registryTLSConfig, ok := c.config.Registry.TLSConfigs[hostname]; ok {
transport.TLSClientConfig, err = c.getTLSConfig(registryTLSConfig)
if err != nil {
return nil, imagespec.Descriptor{}, errors.Wrapf(err, "get TLSConfig for registry %q", refspec.Hostname())
// If the http endpoint is configured, do not try https.
if !util.InStringSlice(endpoints, "http://"+defaultHost) {
if !util.InStringSlice(endpoints, "https://"+defaultHost) {
return append(endpoints, "https://"+defaultHost), nil
}
}
resolver := docker.NewResolver(docker.ResolverOptions{
Credentials: cred,
Client: httpClient,
})
_, desc, err := resolver.Resolve(ctx, ref)
if err != nil {
return nil, imagespec.Descriptor{}, errors.Wrap(err, "no available registry endpoint")
}
return resolver, desc, nil
return endpoints, nil
}
// newTransport returns a new HTTP transport used to pull image.

113
pkg/server/image_pull_test.go
View File

@ -22,8 +22,6 @@ import (
"github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
)
func TestParseAuth(t *testing.T) {
@ -36,6 +34,7 @@ func TestParseAuth(t *testing.T) {
base64.StdEncoding.Encode(invalidAuth, []byte(testUser+"@"+testPasswd))
for desc, test := range map[string]struct {
auth *runtime.AuthConfig
host string
expectedUser string
expectedSecret string
expectErr bool
@ -66,66 +65,92 @@ func TestParseAuth(t *testing.T) {
auth: &runtime.AuthConfig{Auth: string(invalidAuth)},
expectErr: true,
},
"should return empty auth if server address doesn't match": {
auth: &runtime.AuthConfig{
Username: testUser,
Password: testPasswd,
ServerAddress: "https://registry-1.io",
},
host: "registry-2.io",
expectedUser: "",
expectedSecret: "",
},
"should return auth if server address matches": {
auth: &runtime.AuthConfig{
Username: testUser,
Password: testPasswd,
ServerAddress: "https://registry-1.io",
},
host: "registry-1.io",
expectedUser: testUser,
expectedSecret: testPasswd,
},
"should return auth if server address is not specified": {
auth: &runtime.AuthConfig{
Username: testUser,
Password: testPasswd,
},
host: "registry-1.io",
expectedUser: testUser,
expectedSecret: testPasswd,
},
} {
t.Logf("TestCase %q", desc)
u, s, err := ParseAuth(test.auth)
u, s, err := ParseAuth(test.auth, test.host)
assert.Equal(t, test.expectErr, err != nil)
assert.Equal(t, test.expectedUser, u)
assert.Equal(t, test.expectedSecret, s)
}
}
func TestCredentials(t *testing.T) {
c := newTestCRIService()
c.config.Registry.Auths = map[string]criconfig.AuthConfig{
"https://test1.io": {
Username: "username1",
Password: "password1",
},
"http://test2.io": {
Username: "username2",
Password: "password2",
},
"//test3.io": {
Username: "username3",
Password: "password3",
},
}
func TestAddDefaultEndpoint(t *testing.T) {
for desc, test := range map[string]struct {
auth *runtime.AuthConfig
endpoints []string
host string
expectedUsername string
expectedPassword string
expected []string
}{
"auth config from CRI should take precedence": {
auth: &runtime.AuthConfig{
Username: "username",
Password: "password",
"default endpoint not in list": {
endpoints: []string{
"https://registry-1.io",
"https://registry-2.io",
},
host: "test1.io",
expectedUsername: "username",
expectedPassword: "password",
host: "registry-3.io",
expected: []string{
"https://registry-1.io",
"https://registry-2.io",
"https://registry-3.io",
},
"should support https host": {
host: "test1.io",
expectedUsername: "username1",
expectedPassword: "password1",
},
"should support http host": {
host: "test2.io",
expectedUsername: "username2",
expectedPassword: "password2",
"default endpoint in list with http": {
endpoints: []string{
"https://registry-1.io",
"https://registry-2.io",
"http://registry-3.io",
},
host: "registry-3.io",
expected: []string{
"https://registry-1.io",
"https://registry-2.io",
"http://registry-3.io",
},
},
"default endpoint in list with https": {
endpoints: []string{
"https://registry-1.io",
"https://registry-2.io",
"https://registry-3.io",
},
host: "registry-3.io",
expected: []string{
"https://registry-1.io",
"https://registry-2.io",
"https://registry-3.io",
},
"should support hostname only": {
host: "test3.io",
expectedUsername: "username3",
expectedPassword: "password3",
},
} {
t.Logf("TestCase %q", desc)
username, password, err := c.credentials(test.auth)(test.host)
got, err := addDefaultEndpoint(test.endpoints, test.host)
assert.NoError(t, err)
assert.Equal(t, test.expectedUsername, username)
assert.Equal(t, test.expectedPassword, password)
assert.Equal(t, test.expected, got)
}
}