Merge pull request #4138 from dmcgowan/registry-configuration-tools

Registry configuration package

cmd/ctr/commands/commands.go

@@ -61,6 +61,11 @@ var (
 			Name:  "refresh",
 			Usage: "refresh token for authorization server",
 		},
+		cli.StringFlag{
+			Name: "hosts-dir",
+			// compatible with "/etc/docker/certs.d"
+			Usage: "Custom hosts configuration directory",
+		},
 	}
 
 	// ContainerFlags are cli flags specifying container options

cmd/ctr/commands/resolver.go

@@ -21,14 +21,12 @@ import (
 	gocontext "context"
 	"crypto/tls"
 	"fmt"
-	"net"
-	"net/http"
 	"strings"
-	"time"
 
 	"github.com/containerd/console"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/remotes/docker/config"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 )
@@ -60,7 +58,6 @@ func GetResolver(ctx gocontext.Context, clicontext *cli.Context) (remotes.Resolv
 		username = username[0:i]
 	}
 	options := docker.ResolverOptions{
-		PlainHTTP: clicontext.Bool("plain-http"),
 		Tracker: PushTracker,
 	}
 	if username != "" {
@@ -79,32 +76,26 @@ func GetResolver(ctx gocontext.Context, clicontext *cli.Context) (remotes.Resolv
 		secret = rt
 	}
 
-	tr := &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}).DialContext,
-		MaxIdleConns:        10,
-		IdleConnTimeout:     30 * time.Second,
-		TLSHandshakeTimeout: 10 * time.Second,
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: clicontext.Bool("skip-verify"),
-		},
-		ExpectContinueTimeout: 5 * time.Second,
-	}
-
-	options.Client = &http.Client{
-		Transport: tr,
-	}
-
-	credentials := func(host string) (string, string, error) {
+	hostOptions := config.HostOptions{}
+	hostOptions.Credentials = func(host string) (string, string, error) {
+		// If host doesn't match...
 		// Only one host
 		return username, secret, nil
 	}
-	authOpts := []docker.AuthorizerOpt{docker.WithAuthClient(options.Client), docker.WithAuthCreds(credentials)}
-	options.Authorizer = docker.NewDockerAuthorizer(authOpts...)
+	if clicontext.Bool("plain-http") {
+		hostOptions.DefaultScheme = "http"
+	}
+
+	if clicontext.Bool("skip-verify") {
+		hostOptions.DefaultTLS = &tls.Config{
+			InsecureSkipVerify: true,
+		}
+	}
+	if hostDir := clicontext.String("hosts-dir"); hostDir != "" {
+		hostOptions.HostDir = config.HostDirFromRoot(hostDir)
+	}
+
+	options.Hosts = config.ConfigureHosts(ctx, hostOptions)
 
 	return docker.NewResolver(options), nil
 }

remotes/docker/config/config_unix.go

@@ -0,0 +1,40 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package config
+
+import (
+	"crypto/x509"
+	"path/filepath"
+)
+
+func hostPaths(root, host string) []string {
+	ch := hostDirectory(host)
+	if ch == host {
+		return []string{filepath.Join(root, host)}
+	}
+
+	return []string{
+		filepath.Join(root, ch),
+		filepath.Join(root, host),
+	}
+}
+
+func rootSystemPool() (*x509.CertPool, error) {
+	return x509.SystemCertPool()
+}

remotes/docker/config/config_windows.go

@@ -0,0 +1,41 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package config
+
+import (
+	"crypto/x509"
+	"path/filepath"
+	"strings"
+)
+
+func hostPaths(root, host string) []string {
+	ch := hostDirectory(host)
+	if ch == host {
+		return []string{filepath.Join(root, host)}
+	}
+
+	return []string{
+		filepath.Join(root, strings.Replace(ch, ":", "", -1)),
+		filepath.Join(root, strings.Replace(host, ":", "", -1)),
+	}
+}
+
+func rootSystemPool() (*x509.CertPool, error) {
+	return x509.NewCertPool(), nil
+}

remotes/docker/config/hosts.go

@@ -0,0 +1,481 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// config package containers utilities for helping configure the Docker resolver
+package config
+
+import (
+	"context"
+	"crypto/tls"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/pkg/errors"
+)
+
+type hostConfig struct {
+	scheme string
+	host   string
+	path   string
+
+	capabilities docker.HostCapabilities
+
+	caCerts     []string
+	clientPairs [][2]string
+	skipVerify  *bool
+
+	// TODO: API ("docker" or "oci")
+	// TODO: API Version ("v1", "v2")
+	// TODO: Add credential configuration (domain alias, username)
+}
+
+// HostOptions is used to configure registry hosts
+type HostOptions struct {
+	HostDir       func(string) (string, error)
+	Credentials   func(host string) (string, string, error)
+	DefaultTLS    *tls.Config
+	DefaultScheme string
+}
+
+// ConfigureHosts creates a registry hosts function from the provided
+// host creation options. The host directory can read hosts.toml or
+// certificate files laid out in the Docker specific layout.
+// If a `HostDir` function is not required, defaults are used.
+func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHosts {
+	return func(host string) ([]docker.RegistryHost, error) {
+		var hosts []hostConfig
+		if options.HostDir != nil {
+			dir, err := options.HostDir(host)
+			if err != nil && !errdefs.IsNotFound(err) {
+				return nil, err
+			}
+			if dir != "" {
+				hosts, err = loadHostDir(ctx, dir)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+		}
+
+		// If hosts was not set, add a default host
+		// NOTE: Check nil here and not empty, the host may be
+		// intentionally configured to not have any endpoints
+		if hosts == nil {
+			hosts = make([]hostConfig, 1)
+		}
+		if len(hosts) > 1 {
+			if hosts[len(hosts)-1].host == "" {
+				if host == "docker.io" {
+					hosts[len(hosts)-1].scheme = "https"
+					hosts[len(hosts)-1].host = "https://registry-1.docker.io"
+				} else {
+					hosts[len(hosts)-1].host = host
+					if options.DefaultScheme != "" {
+						hosts[len(hosts)-1].scheme = options.DefaultScheme
+					} else {
+						hosts[len(hosts)-1].scheme = "https"
+					}
+				}
+				hosts[len(hosts)-1].path = "/v2"
+			}
+		}
+
+		defaultTransport := &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          10,
+			IdleConnTimeout:       30 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			TLSClientConfig:       options.DefaultTLS,
+			ExpectContinueTimeout: 5 * time.Second,
+		}
+
+		client := &http.Client{
+			Transport: defaultTransport,
+		}
+
+		authOpts := []docker.AuthorizerOpt{docker.WithAuthClient(client)}
+		if options.Credentials != nil {
+			authOpts = append(authOpts, docker.WithAuthCreds(options.Credentials))
+		}
+		authorizer := docker.NewDockerAuthorizer(authOpts...)
+
+		rhosts := make([]docker.RegistryHost, len(hosts))
+		for i, host := range hosts {
+
+			rhosts[i].Scheme = host.scheme
+			rhosts[i].Host = host.host
+			rhosts[i].Path = host.path
+			rhosts[i].Capabilities = host.capabilities
+
+			if host.caCerts != nil || host.clientPairs != nil || host.skipVerify != nil {
+				var tlsConfig *tls.Config
+				if options.DefaultTLS != nil {
+					tlsConfig = options.DefaultTLS.Clone()
+				} else {
+					tlsConfig = &tls.Config{}
+				}
+				if host.skipVerify != nil {
+					tlsConfig.InsecureSkipVerify = *host.skipVerify
+				}
+				if host.caCerts != nil {
+					if tlsConfig.RootCAs == nil {
+						rootPool, err := rootSystemPool()
+						if err != nil {
+							return nil, errors.Wrap(err, "unable to initialize cert pool")
+						}
+						tlsConfig.RootCAs = rootPool
+					}
+					for _, f := range host.caCerts {
+						data, err := ioutil.ReadFile(f)
+						if err != nil {
+							return nil, errors.Wrapf(err, "unable to read CA cert %q", f)
+						}
+						if !tlsConfig.RootCAs.AppendCertsFromPEM(data) {
+							return nil, errors.Errorf("unable to load CA cert %q", f)
+						}
+					}
+				}
+
+				if host.clientPairs != nil {
+					for _, pair := range host.clientPairs {
+						certPEMBlock, err := ioutil.ReadFile(pair[0])
+						if err != nil {
+							return nil, errors.Wrapf(err, "unable to read CERT file %q", pair[0])
+						}
+						var keyPEMBlock []byte
+						if pair[1] != "" {
+							keyPEMBlock, err = ioutil.ReadFile(pair[1])
+							if err != nil {
+								return nil, errors.Wrapf(err, "unable to read CERT file %q", pair[1])
+							}
+						} else {
+							// Load key block from same PEM file
+							keyPEMBlock = certPEMBlock
+						}
+						cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+						if err != nil {
+							return nil, errors.Wrap(err, "failed to load X509 key pair")
+						}
+
+						tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
+					}
+				}
+				tr := defaultTransport.Clone()
+				tr.TLSClientConfig = tlsConfig
+				c := *client
+				c.Transport = tr
+
+				rhosts[i].Client = &c
+				rhosts[i].Authorizer = docker.NewDockerAuthorizer(append(authOpts, docker.WithAuthClient(&c))...)
+			} else {
+				rhosts[i].Client = client
+				rhosts[i].Authorizer = authorizer
+			}
+
+		}
+
+		return rhosts, nil
+	}
+
+}
+
+// HostDirFromRoot returns a function which finds a host directory
+// based at the given root.
+func HostDirFromRoot(root string) func(string) (string, error) {
+	return func(host string) (string, error) {
+		for _, p := range hostPaths(root, host) {
+			if _, err := os.Stat(p); err == nil {
+				return p, nil
+			} else if !os.IsNotExist(err) {
+				return "", err
+			}
+		}
+		return "", errdefs.ErrNotFound
+	}
+}
+
+// hostDirectory converts ":port" to "_port_" in directory names
+func hostDirectory(host string) string {
+	idx := strings.LastIndex(host, ":")
+	if idx > 0 {
+		return host[:idx] + "_" + host[idx+1:] + "_"
+	}
+	return host
+}
+
+func loadHostDir(ctx context.Context, hostsDir string) ([]hostConfig, error) {
+	b, err := ioutil.ReadFile(filepath.Join(hostsDir, "hosts.toml"))
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	if len(b) == 0 {
+		// If hosts.toml does not exist, fallback to checking for
+		// certificate files based on Docker's certificate file
+		// pattern (".crt", ".cert", ".key" files)
+		return loadCertFiles(ctx, hostsDir)
+	}
+
+	hosts, err := parseHostsFile(ctx, hostsDir, b)
+	if err != nil {
+		log.G(ctx).WithError(err).Error("failed to decode hosts.toml")
+		// Fallback to checking certificate files
+		return loadCertFiles(ctx, hostsDir)
+	}
+
+	return hosts, nil
+}
+
+type hostFileConfig struct {
+	// Capabilities determine what operations a host is
+	// capable of performing. Allowed values
+	//  - pull
+	//  - resolve
+	//  - push
+	Capabilities []string `toml:"capabilities"`
+
+	// CACert can be a string or an array of strings
+	CACert toml.Primitive `toml:"ca"`
+
+	// TODO: Make this an array (two key types, one for pairs (multiple files), one for single file?)
+	Client toml.Primitive `toml:"client"`
+
+	SkipVerify bool `toml:"skip_verify"`
+
+	// API (default: "docker")
+	// API Version (default: "v2")
+	// Credentials: helper? name? username? alternate domain? token?
+}
+
+type configFile struct {
+	// hostConfig holds defaults for all hosts as well as
+	// for the default server
+	hostFileConfig
+
+	// Server specifies the default server. When `host` is
+	// also specified, those hosts are tried first.
+	Server string `toml:"server"`
+
+	// HostConfigs store the per-host configuration
+	HostConfigs map[string]hostFileConfig `toml:"host"`
+}
+
+func parseHostsFile(ctx context.Context, baseDir string, b []byte) ([]hostConfig, error) {
+	var c configFile
+	md, err := toml.Decode(string(b), &c)
+	if err != nil {
+		return nil, err
+	}
+
+	var orderedHosts []string
+	for _, key := range md.Keys() {
+		if len(key) >= 2 {
+			if key[0] == "host" && (len(orderedHosts) == 0 || orderedHosts[len(orderedHosts)-1] != key[1]) {
+				orderedHosts = append(orderedHosts, key[1])
+			}
+		}
+	}
+
+	if c.Server != "" {
+		c.HostConfigs[c.Server] = c.hostFileConfig
+		orderedHosts = append(orderedHosts, c.Server)
+	} else if len(orderedHosts) == 0 {
+		c.HostConfigs[""] = c.hostFileConfig
+		orderedHosts = append(orderedHosts, "")
+	}
+	hosts := make([]hostConfig, len(orderedHosts))
+	for i, server := range orderedHosts {
+		hostConfig := c.HostConfigs[server]
+
+		if !strings.HasPrefix(server, "http") {
+			server = "https://" + server
+		}
+		u, err := url.Parse(server)
+		if err != nil {
+			return nil, errors.Errorf("unable to parse server %v", server)
+		}
+		hosts[i].scheme = u.Scheme
+		hosts[i].host = u.Host
+
+		// TODO: Handle path based on registry protocol
+		// Define a registry protocol type
+		//   OCI v1    - Always use given path as is
+		//   Docker v2 - Always ensure ends with /v2/
+		if len(u.Path) > 0 {
+			u.Path = path.Clean(u.Path)
+			if !strings.HasSuffix(u.Path, "/v2") {
+				u.Path = u.Path + "/v2"
+			}
+		} else {
+			u.Path = "/v2"
+		}
+		hosts[i].path = u.Path
+
+		if hosts[i].scheme == "https" {
+			hosts[i].skipVerify = &hostConfig.SkipVerify
+		}
+
+		if len(hostConfig.Capabilities) > 0 {
+			for _, c := range hostConfig.Capabilities {
+				switch strings.ToLower(c) {
+				case "pull":
+					hosts[i].capabilities |= docker.HostCapabilityPull
+				case "resolve":
+					hosts[i].capabilities |= docker.HostCapabilityResolve
+				case "push":
+					hosts[i].capabilities |= docker.HostCapabilityPush
+				default:
+					return nil, errors.Errorf("unknown capability %v", c)
+				}
+			}
+		} else {
+			hosts[i].capabilities = docker.HostCapabilityPull | docker.HostCapabilityResolve | docker.HostCapabilityPush
+		}
+
+		baseKey := []string{}
+		if server != "" {
+			baseKey = append(baseKey, "host", server)
+		}
+		caKey := append(baseKey, "ca")
+		if md.IsDefined(caKey...) {
+			switch t := md.Type(caKey...); t {
+			case "String":
+				var caCert string
+				if err := md.PrimitiveDecode(hostConfig.CACert, &caCert); err != nil {
+					return nil, errors.Wrap(err, "failed to decode \"ca\"")
+				}
+				hosts[i].caCerts = []string{makeAbsPath(caCert, baseDir)}
+			case "Array":
+				var caCerts []string
+				if err := md.PrimitiveDecode(hostConfig.CACert, &caCerts); err != nil {
+					return nil, errors.Wrap(err, "failed to decode \"ca\"")
+				}
+				for i, p := range caCerts {
+					caCerts[i] = makeAbsPath(p, baseDir)
+				}
+
+				hosts[i].caCerts = caCerts
+			default:
+				return nil, errors.Errorf("invalid type %v for \"ca\"", t)
+			}
+		}
+
+		clientKey := append(baseKey, "client")
+		if md.IsDefined(clientKey...) {
+			switch t := md.Type(clientKey...); t {
+			case "String":
+				var clientCert string
+				if err := md.PrimitiveDecode(hostConfig.Client, &clientCert); err != nil {
+					return nil, errors.Wrap(err, "failed to decode \"ca\"")
+				}
+				hosts[i].clientPairs = [][2]string{{makeAbsPath(clientCert, baseDir), ""}}
+			case "Array":
+				var clientCerts []interface{}
+				if err := md.PrimitiveDecode(hostConfig.Client, &clientCerts); err != nil {
+					return nil, errors.Wrap(err, "failed to decode \"ca\"")
+				}
+				for _, pairs := range clientCerts {
+					switch p := pairs.(type) {
+					case string:
+						hosts[i].clientPairs = append(hosts[i].clientPairs, [2]string{makeAbsPath(p, baseDir), ""})
+					case []interface{}:
+						var pair [2]string
+						if len(p) > 2 {
+							return nil, errors.Errorf("invalid pair %v for \"client\"", p)
+						}
+						for pi, cp := range p {
+							s, ok := cp.(string)
+							if !ok {
+								return nil, errors.Errorf("invalid type %T for \"client\"", cp)
+							}
+							pair[pi] = makeAbsPath(s, baseDir)
+						}
+						hosts[i].clientPairs = append(hosts[i].clientPairs, pair)
+					default:
+						return nil, errors.Errorf("invalid type %T for \"client\"", p)
+					}
+				}
+			default:
+				return nil, errors.Errorf("invalid type %v for \"client\"", t)
+			}
+		}
+	}
+
+	return hosts, nil
+}
+
+func makeAbsPath(p string, base string) string {
+	if filepath.IsAbs(p) {
+		return p
+	}
+	return filepath.Join(base, p)
+}
+
+// loadCertsDir loads certs from certsDir like "/etc/docker/certs.d" .
+// Compatible with Docker file layout
+// - files ending with ".crt" are treated as CA certificate files
+// - files ending with ".cert" are treated as client certificates, and
+//   files with the same name but ending with ".key" are treated as the
+//   corresponding private key.
+//   NOTE: If a ".key" file is missing, this function will just return
+//   the ".cert", which may contain the private key. If the ".cert" file
+//   does not contain the private key, the caller should detect and error.
+func loadCertFiles(ctx context.Context, certsDir string) ([]hostConfig, error) {
+	fs, err := ioutil.ReadDir(certsDir)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	hosts := make([]hostConfig, 1)
+	for _, f := range fs {
+		if !f.IsDir() {
+			continue
+		}
+		if strings.HasSuffix(f.Name(), ".crt") {
+			hosts[0].caCerts = append(hosts[0].caCerts, filepath.Join(certsDir, f.Name()))
+		}
+		if strings.HasSuffix(f.Name(), ".cert") {
+			var pair [2]string
+			certFile := f.Name()
+			pair[0] = filepath.Join(certsDir, certFile)
+			// Check if key also exists
+			keyFile := certFile[:len(certFile)-5] + ".key"
+			if _, err := os.Stat(keyFile); err == nil {
+				pair[1] = filepath.Join(certsDir, keyFile)
+			} else if !os.IsNotExist(err) {
+				return nil, err
+			}
+			hosts[0].clientPairs = append(hosts[0].clientPairs, pair)
+		}
+	}
+	return hosts, nil
+}

remotes/docker/config/hosts_test.go

@@ -0,0 +1,205 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package config
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/containerd/remotes/docker"
+)
+
+func TestParseHostFile(t *testing.T) {
+	ctx := context.Background()
+
+	const testtoml = `
+server = "https://test-default.registry"
+ca = "/etc/path/default"
+
+[host."https://mirror.registry"]
+  capabilities = ["pull"]
+  ca = "/etc/certs/mirror.pem"
+
+[host."https://mirror-bak.registry/us"]
+  capabilities = ["pull"]
+  skip_verify = true
+
+[host."http://mirror.registry"]
+  capabilities = ["pull"]
+
+[host."https://test-1.registry"]
+  capabilities = ["pull", "resolve", "push"]
+  ca = ["/etc/certs/test-1-ca.pem", "/etc/certs/special.pem"]
+  client = [["/etc/certs/client.cert", "/etc/certs/client.key"],["/etc/certs/client.pem", ""]]
+
+[host."https://test-2.registry"]
+  client = "/etc/certs/client.pem"
+
+[host."https://test-3.registry"]
+  client = ["/etc/certs/client-1.pem", "/etc/certs/client-2.pem"]
+`
+	var tb, fb = true, false
+	var allCaps = docker.HostCapabilityPull | docker.HostCapabilityResolve | docker.HostCapabilityPush
+	expected := []hostConfig{
+		{
+			scheme:       "https",
+			host:         "mirror.registry",
+			path:         "/v2",
+			capabilities: docker.HostCapabilityPull,
+			caCerts:      []string{filepath.FromSlash("/etc/certs/mirror.pem")},
+			skipVerify:   &fb,
+		},
+		{
+			scheme:       "https",
+			host:         "mirror-bak.registry",
+			path:         "/us/v2",
+			capabilities: docker.HostCapabilityPull,
+			skipVerify:   &tb,
+		},
+		{
+			scheme:       "http",
+			host:         "mirror.registry",
+			path:         "/v2",
+			capabilities: docker.HostCapabilityPull,
+		},
+		{
+			scheme:       "https",
+			host:         "test-1.registry",
+			path:         "/v2",
+			capabilities: allCaps,
+			caCerts:      []string{filepath.FromSlash("/etc/certs/test-1-ca.pem"), filepath.FromSlash("/etc/certs/special.pem")},
+			clientPairs: [][2]string{
+				{filepath.FromSlash("/etc/certs/client.cert"), filepath.FromSlash("/etc/certs/client.key")},
+				{filepath.FromSlash("/etc/certs/client.pem"), ""},
+			},
+			skipVerify: &fb,
+		},
+		{
+			scheme:       "https",
+			host:         "test-2.registry",
+			path:         "/v2",
+			capabilities: allCaps,
+			clientPairs: [][2]string{
+				{filepath.FromSlash("/etc/certs/client.pem")},
+			},
+			skipVerify: &fb,
+		},
+		{
+			scheme:       "https",
+			host:         "test-3.registry",
+			path:         "/v2",
+			capabilities: allCaps,
+			clientPairs: [][2]string{
+				{filepath.FromSlash("/etc/certs/client-1.pem")},
+				{filepath.FromSlash("/etc/certs/client-2.pem")},
+			},
+			skipVerify: &fb,
+		},
+		{
+			scheme:       "https",
+			host:         "test-default.registry",
+			path:         "/v2",
+			capabilities: allCaps,
+			skipVerify:   &fb,
+		},
+	}
+	hosts, err := parseHostsFile(ctx, "", []byte(testtoml))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if t.Failed() {
+			t.Log("HostConfigs...\nActual:\n" + printHostConfig(hosts) + "Expected:\n" + printHostConfig(expected))
+		}
+	}()
+
+	if len(hosts) != len(expected) {
+		t.Fatalf("Unexpected number of hosts %d, expected %d", len(hosts), len(expected))
+	}
+
+	for i := range hosts {
+		if !compareHostConfig(hosts[i], expected[i]) {
+			t.Fatalf("Mismatch at host %d", i)
+		}
+	}
+}
+
+func compareHostConfig(j, k hostConfig) bool {
+	if j.scheme != k.scheme {
+		return false
+	}
+	if j.host != k.host {
+		return false
+	}
+	if j.path != k.path {
+		return false
+	}
+	if j.capabilities != k.capabilities {
+		return false
+	}
+
+	if len(j.caCerts) != len(k.caCerts) {
+		return false
+	}
+	for i := range j.caCerts {
+		if j.caCerts[i] != k.caCerts[i] {
+			return false
+		}
+	}
+	if len(j.clientPairs) != len(k.clientPairs) {
+		return false
+	}
+	for i := range j.clientPairs {
+		if j.clientPairs[i][0] != k.clientPairs[i][0] {
+			return false
+		}
+		if j.clientPairs[i][1] != k.clientPairs[i][1] {
+			return false
+		}
+	}
+	if j.skipVerify != nil && k.skipVerify != nil {
+		if *j.skipVerify != *k.skipVerify {
+			return false
+		}
+	} else if j.skipVerify != nil || k.skipVerify != nil {
+		return false
+	}
+
+	return true
+}
+
+func printHostConfig(hc []hostConfig) string {
+	b := bytes.NewBuffer(nil)
+	for i := range hc {
+		fmt.Fprintf(b, "\t[%d]\tscheme: %q\n", i, hc[i].scheme)
+		fmt.Fprintf(b, "\t\thost: %q\n", hc[i].host)
+		fmt.Fprintf(b, "\t\tpath: %q\n", hc[i].path)
+		fmt.Fprintf(b, "\t\tcaps: %03b\n", hc[i].capabilities)
+		fmt.Fprintf(b, "\t\tca: %#v\n", hc[i].caCerts)
+		fmt.Fprintf(b, "\t\tclients: %#v\n", hc[i].clientPairs)
+		if hc[i].skipVerify == nil {
+			fmt.Fprintf(b, "\t\tskip-verify: %v\n", hc[i].skipVerify)
+		} else {
+			fmt.Fprintf(b, "\t\tskip-verify: %t\n", *hc[i].skipVerify)
+		}
+	}
+	return b.String()
+}