Merge pull request #4978 from cpuguy83/certs_dir

Add support for using a host registry dir in cri

docs/cri/config.md

@@ -237,6 +237,11 @@ version = 2
 
   # 'plugins."io.containerd.grpc.v1.cri".registry' contains config related to the registry
   [plugins."io.containerd.grpc.v1.cri".registry]
+    # Specifies a directory to look for registry configs in.
+    # Dir can be used just like /etc/docker/certs.d OR can contain a hosts.toml with more specific configurations.
+    #
+    # NOTE: Specifying this will cause the cri plugin to ignore any other registry configs specified in this configuration file.
+    config_path = "/etc/containerd/certs.d"
 
     # 'plugins."io.containerd.grpc.v1.cri.registry.headers sets the http request headers to send for all registry requests
     [plugins."io.containerd.grpc.v1.cri".registry.headers]

pkg/cri/config/config.go

@@ -145,15 +145,21 @@ type TLSConfig struct {
 
 // Registry is registry settings configured
 type Registry struct {
+	// ConfigPath is a path to the root directory containing registry-specific
+	// configurations.
+	// If ConfigPath is set, the rest of the registry specific options are ignored.
+	ConfigPath string `toml:"config_path" json:"configPath"`
 	// Mirrors are namespace to mirror mapping for all namespaces.
+	// This option will not be used when ConfigPath is provided.
+	// DEPRECATED: Use ConfigPath instead. Remove in containerd 1.7.
 	Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"`
 	// Configs are configs for each registry.
 	// The key is the domain name or IP of the registry.
+	// This option will be fully deprecated for ConfigPath in the future.
 	Configs map[string]RegistryConfig `toml:"configs" json:"configs"`
-
 	// Auths are registry endpoint to auth config mapping. The registry endpoint must
 	// be a valid url with host specified.
-	// DEPRECATED: Use Configs instead. Remove in containerd 1.4.
+	// DEPRECATED: Use ConfigPath instead. Remove in containerd 1.6.
 	Auths map[string]AuthConfig `toml:"auths" json:"auths"`
 	// Headers adds additional HTTP headers that get sent to all registries
 	Headers map[string][]string `toml:"headers" json:"headers"`
@@ -165,6 +171,8 @@ type RegistryConfig struct {
 	Auth *AuthConfig `toml:"auth" json:"auth"`
 	// TLS is a pair of CA/Cert/Key which then are used when creating the transport
 	// that communicates with the registry.
+	// This field will not be used when ConfigPath is provided.
+	// DEPRECATED: Use ConfigPath instead. Remove in containerd 1.7.
 	TLS *TLSConfig `toml:"tls" json:"tls"`
 }
 
@@ -351,6 +359,27 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
 		}
 	}
 
+	useConfigPath := c.Registry.ConfigPath != ""
+	if len(c.Registry.Mirrors) > 0 {
+		if useConfigPath {
+			return errors.Errorf("`mirrors` cannot be set when `config_path` is provided")
+		}
+		log.G(ctx).Warning("`mirrors` is deprecated, please use `config_path` instead")
+	}
+	var hasDeprecatedTLS bool
+	for _, r := range c.Registry.Configs {
+		if r.TLS != nil {
+			hasDeprecatedTLS = true
+			break
+		}
+	}
+	if hasDeprecatedTLS {
+		if useConfigPath {
+			return errors.Errorf("`configs.tls` cannot be set when `config_path` is provided")
+		}
+		log.G(ctx).Warning("`configs.tls` is deprecated, please use `config_path` instead")
+	}
+
 	// Validation for deprecated auths options and mapping it to configs.
 	if len(c.Registry.Auths) != 0 {
 		if c.Registry.Configs == nil {
@@ -370,7 +399,7 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
 			config.Auth = &auth
 			c.Registry.Configs[endpoint] = config
 		}
-		log.G(ctx).Warning("`auths` is deprecated, please use registry`configs` instead")
+		log.G(ctx).Warning("`auths` is deprecated, please use `configs` instead")
 	}
 
 	// Validation for stream_idle_timeout

pkg/cri/config/config_test.go

@@ -320,6 +320,46 @@ func TestValidateConfig(t *testing.T) {
 			},
 			expectedErr: "invalid stream idle timeout",
 		},
+		"conflicting mirror registry config": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: "default",
+						},
+					},
+				},
+				Registry: Registry{
+					ConfigPath: "/etc/containerd/conf.d",
+					Mirrors: map[string]Mirror{
+						"something.io": {},
+					},
+				},
+			},
+			expectedErr: "`mirrors` cannot be set when `config_path` is provided",
+		},
+		"conflicting tls registry config": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: "default",
+						},
+					},
+				},
+				Registry: Registry{
+					ConfigPath: "/etc/containerd/conf.d",
+					Configs: map[string]RegistryConfig{
+						"something.io": {
+							TLS: &TLSConfig{},
+						},
+					},
+				},
+			},
+			expectedErr: "`configs.tls` cannot be set when `config_path` is provided",
+		},
 	} {
 		t.Run(desc, func(t *testing.T) {
 			err := ValidatePluginConfig(context.Background(), test.config)

pkg/cri/server/image_pull.go

@@ -25,6 +25,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -35,6 +36,7 @@ import (
 	"github.com/containerd/containerd/log"
 	distribution "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/remotes/docker/config"
 	"github.com/containerd/imgcrypt"
 	"github.com/containerd/imgcrypt/images/encryption"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -100,7 +102,7 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
 	var (
 		resolver = docker.NewResolver(docker.ResolverOptions{
 			Headers: c.config.Registry.Headers,
-			Hosts:   c.registryHosts(r.GetAuth()),
+			Hosts:   c.registryHosts(ctx, r.GetAuth()),
 		})
 		isSchema1    bool
 		imageHandler containerdimages.HandlerFunc = func(_ context.Context,
@@ -311,8 +313,42 @@ func (c *criService) getTLSConfig(registryTLSConfig criconfig.TLSConfig) (*tls.C
 	return tlsConfig, nil
 }
 
+func hostDirFromRoots(roots []string) func(string) (string, error) {
+	rootfn := make([]func(string) (string, error), len(roots))
+	for i := range roots {
+		rootfn[i] = config.HostDirFromRoot(roots[i])
+	}
+	return func(host string) (dir string, err error) {
+		for _, fn := range rootfn {
+			dir, err = fn(host)
+			if (err != nil && !errdefs.IsNotFound(err)) || (dir != "") {
+				break
+			}
+		}
+		return
+	}
+}
+
 // registryHosts is the registry hosts to be used by the resolver.
-func (c *criService) registryHosts(auth *runtime.AuthConfig) docker.RegistryHosts {
+func (c *criService) registryHosts(ctx context.Context, auth *runtime.AuthConfig) docker.RegistryHosts {
+	paths := filepath.SplitList(c.config.Registry.ConfigPath)
+	if len(paths) > 0 {
+		hostOptions := config.HostOptions{}
+		hostOptions.Credentials = func(host string) (string, string, error) {
+			hostauth := auth
+			if hostauth == nil {
+				config := c.config.Registry.Configs[host]
+				if config.Auth != nil {
+					hostauth = toRuntimeAuthConfig(*config.Auth)
+				}
+			}
+			return ParseAuth(hostauth, host)
+		}
+		hostOptions.HostDir = hostDirFromRoots(paths)
+
+		return config.ConfigureHosts(ctx, hostOptions)
+	}
+
 	return func(host string) ([]docker.RegistryHost, error) {
 		var registries []docker.RegistryHost