From dc131aa862cfb41b3bc866e7af4ff8e8cd739b3d Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 10 Aug 2019 17:07:58 +0900 Subject: [PATCH 1/3] support loading certs from a directory Add `remotes/certutil` functions for loading `ca.crt`, `client.cert`, and `client.key` into `tls.Config` from a directory like `/etc/docker/certs.d/. See https://docs.docker.com/engine/security/certificates/ . Client applications including CRI plugin are expected to configure the resolver using these functions. As an example, the `ctr` tool is extended to support `ctr images pull --certs-dir=/etc/docker/certs.d example.com/foo/bar:baz`. Tested with Harbor 1.8. Signed-off-by: Akihiro Suda --- cmd/ctr/commands/commands.go | 5 ++ cmd/ctr/commands/resolver.go | 33 +++++++++ remotes/certutil/certutil.go | 126 +++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 remotes/certutil/certutil.go diff --git a/cmd/ctr/commands/commands.go b/cmd/ctr/commands/commands.go index 808ea37d7..253987730 100644 --- a/cmd/ctr/commands/commands.go +++ b/cmd/ctr/commands/commands.go @@ -61,6 +61,11 @@ var ( Name: "refresh", Usage: "refresh token for authorization server", }, + cli.StringFlag{ + Name: "certs-dir", + // compatible with "/etc/docker/certs.d" + Usage: "custom certificates directory that contains \"/{ca.crt, client.cert, client.key}\"", + }, } // ContainerFlags are cli flags specifying container options diff --git a/cmd/ctr/commands/resolver.go b/cmd/ctr/commands/resolver.go index 1b3c30a86..5f4b11257 100644 --- a/cmd/ctr/commands/resolver.go +++ b/cmd/ctr/commands/resolver.go @@ -21,15 +21,20 @@ import ( gocontext "context" "crypto/tls" "fmt" + "io/ioutil" "net" "net/http" + "os" + "path/filepath" "strings" "time" "github.com/containerd/console" "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/certutil" "github.com/containerd/containerd/remotes/docker" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -94,6 +99,7 @@ func GetResolver(ctx gocontext.Context, clicontext *cli.Context) (remotes.Resolv }, ExpectContinueTimeout: 5 * time.Second, } + loadCertsDir(tr.TLSClientConfig, clicontext.String("certs-dir")) options.Client = &http.Client{ Transport: tr, @@ -108,3 +114,30 @@ func GetResolver(ctx gocontext.Context, clicontext *cli.Context) (remotes.Resolv return docker.NewResolver(options), nil } + +// loadCertsDir loads certs from certsDir like "/etc/docker/certs.d" . +func loadCertsDir(config *tls.Config, certsDir string) { + if certsDir == "" { + return + } + fs, err := ioutil.ReadDir(certsDir) + if err != nil && !os.IsNotExist(err) { + logrus.WithError(err).Errorf("cannot read certs directory %q", certsDir) + return + } + for _, f := range fs { + if !f.IsDir() { + continue + } + // TODO: skip loading if f.Name() is not valid FQDN/IP + hostDir := filepath.Join(certsDir, f.Name()) + caCertGlob := filepath.Join(hostDir, "*.crt") + if _, err = certutil.LoadCACerts(config, caCertGlob); err != nil { + logrus.WithError(err).Errorf("cannot load certs from %q", caCertGlob) + } + keyGlob := filepath.Join(hostDir, "*.key") + if _, _, err = certutil.LoadKeyPairs(config, keyGlob, certutil.DockerKeyPairCertLocator); err != nil { + logrus.WithError(err).Errorf("cannot load key pairs from %q", keyGlob) + } + } +} diff --git a/remotes/certutil/certutil.go b/remotes/certutil/certutil.go new file mode 100644 index 000000000..7755388b3 --- /dev/null +++ b/remotes/certutil/certutil.go @@ -0,0 +1,126 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package certutil + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/containerd/containerd/errdefs" + "github.com/pkg/errors" +) + +// SystemCertPool returns a copy of the system cert pool, +// returns an error if failed to load or empty pool on windows. +// +// SystemCertPool was ported from Docker 19.03 +// https://github.com/docker/engine/blob/v19.03.1/vendor/github.com/docker/go-connections/tlsconfig/certpool_go17.go#L12 +func SystemCertPool() (*x509.CertPool, error) { + certpool, err := x509.SystemCertPool() + if err != nil && runtime.GOOS == "windows" { + return x509.NewCertPool(), nil + } + return certpool, err +} + +// LoadCACerts loads CA certificates into tlsConfig from glob. +// glob should be like "/etc/docker/certs.d/example.com/*.crt" . +// LoadCACerts returns the paths of the loaded certs. +func LoadCACerts(tlsConfig *tls.Config, glob string) ([]string, error) { + if tlsConfig == nil { + tlsConfig = &tls.Config{} + } + files, err := filepath.Glob(glob) + if err != nil { + return nil, err + } + sort.Strings(files) + if tlsConfig.RootCAs == nil { + systemPool, err := SystemCertPool() + if err != nil { + return nil, errors.Wrap(err, "unable to get system cert poolv") + } + tlsConfig.RootCAs = systemPool + } + var loaded []string + for _, f := range files { + data, err := ioutil.ReadFile(f) + if err != nil { + return loaded, errors.Wrapf(err, "unable to read CA cert %q", f) + } + if !tlsConfig.RootCAs.AppendCertsFromPEM(data) { + return loaded, errors.Errorf("unable to load CA cert %q", f) + } + loaded = append(loaded, f) + } + return loaded, nil +} + +// KeyPairCertLocator is used to resolve the cert path from the key path. +type KeyPairCertLocator func(keyPath string) (certPath string, err error) + +// DockerKeyPairCertLocator implements the Docker-style convention. ("*.key" -> "*.cert") +var DockerKeyPairCertLocator KeyPairCertLocator = func(keyPath string) (string, error) { + if !strings.HasSuffix(keyPath, ".key") { + return "", errors.Errorf("expected key path with \".key\" suffix, got %q", keyPath) + } + // Docker convention uses *.crt for CA certs, *.cert for keypair certs. + certPath := keyPath[:len(keyPath)-4] + ".cert" + return certPath, nil +} + +// LoadKeyPairs loads key pairs into tlsConfig from keyGlob. +// keyGlob should be like "/etc/docker/certs.d/example.com/*.key" . +// certLocator is used to resolve the cert path from the key path. +// Use DockerKeyPairCertLocator for the Docker-style convention. ("*.key" -> "*.cert") +// LoadKeyParis returns the paths of the loaded certs and the loaded keys. +func LoadKeyPairs(tlsConfig *tls.Config, keyGlob string, certLocator KeyPairCertLocator) ([]string, []string, error) { + if tlsConfig == nil { + tlsConfig = &tls.Config{} + } + if certLocator == nil { + return nil, nil, errors.Wrap(errdefs.ErrInvalidArgument, "missing cert locator") + } + keyPaths, err := filepath.Glob(keyGlob) + if err != nil { + return nil, nil, err + } + sort.Strings(keyPaths) + var ( + loadedCerts []string + loadedKeys []string + ) + for _, keyPath := range keyPaths { + certPath, err := certLocator(keyPath) + if err != nil { + return loadedCerts, loadedKeys, err + } + keyPair, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return loadedCerts, loadedKeys, err + } + tlsConfig.Certificates = append(tlsConfig.Certificates, keyPair) + loadedCerts = append(loadedCerts, certPath) + loadedKeys = append(loadedKeys, keyPath) + } + return loadedCerts, loadedKeys, nil +} From 17b6050d20bf7d25d15e26ed867f774e8c7c72fe Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 26 Sep 2019 10:30:12 -0700 Subject: [PATCH 2/3] Add Docker resolver configuration package Add configuration toml file format and configuration function to configure registry hosts from a directory based configuration. Compatible with Docker registry certificate loading. Signed-off-by: Derek McGowan --- remotes/docker/config/config_unix.go | 40 ++ remotes/docker/config/config_windows.go | 41 ++ remotes/docker/config/hosts.go | 481 ++++++++++++++++++++++++ remotes/docker/config/hosts_test.go | 205 ++++++++++ 4 files changed, 767 insertions(+) create mode 100644 remotes/docker/config/config_unix.go create mode 100644 remotes/docker/config/config_windows.go create mode 100644 remotes/docker/config/hosts.go create mode 100644 remotes/docker/config/hosts_test.go diff --git a/remotes/docker/config/config_unix.go b/remotes/docker/config/config_unix.go new file mode 100644 index 000000000..8245c4d70 --- /dev/null +++ b/remotes/docker/config/config_unix.go @@ -0,0 +1,40 @@ +// +build !windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "crypto/x509" + "path/filepath" +) + +func hostPaths(root, host string) []string { + ch := hostDirectory(host) + if ch == host { + return []string{filepath.Join(root, host)} + } + + return []string{ + filepath.Join(root, ch), + filepath.Join(root, host), + } +} + +func rootSystemPool() (*x509.CertPool, error) { + return x509.SystemCertPool() +} diff --git a/remotes/docker/config/config_windows.go b/remotes/docker/config/config_windows.go new file mode 100644 index 000000000..948b65396 --- /dev/null +++ b/remotes/docker/config/config_windows.go @@ -0,0 +1,41 @@ +// +build windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "crypto/x509" + "path/filepath" + "strings" +) + +func hostPaths(root, host string) []string { + ch := hostDirectory(host) + if ch == host { + return []string{filepath.Join(root, host)} + } + + return []string{ + filepath.Join(root, strings.Replace(ch, ":", "", -1)), + filepath.Join(root, strings.Replace(host, ":", "", -1)), + } +} + +func rootSystemPool() (*x509.CertPool, error) { + return x509.NewCertPool(), nil +} diff --git a/remotes/docker/config/hosts.go b/remotes/docker/config/hosts.go new file mode 100644 index 000000000..b9736cafb --- /dev/null +++ b/remotes/docker/config/hosts.go @@ -0,0 +1,481 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// config package containers utilities for helping configure the Docker resolver +package config + +import ( + "context" + "crypto/tls" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/BurntSushi/toml" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/remotes/docker" + "github.com/pkg/errors" +) + +type hostConfig struct { + scheme string + host string + path string + + capabilities docker.HostCapabilities + + caCerts []string + clientPairs [][2]string + skipVerify *bool + + // TODO: API ("docker" or "oci") + // TODO: API Version ("v1", "v2") + // TODO: Add credential configuration (domain alias, username) +} + +// HostOptions is used to configure registry hosts +type HostOptions struct { + HostDir func(string) (string, error) + Credentials func(host string) (string, string, error) + DefaultTLS *tls.Config + DefaultScheme string +} + +// ConfigureHosts creates a registry hosts function from the provided +// host creation options. The host directory can read hosts.toml or +// certificate files laid out in the Docker specific layout. +// If a `HostDir` function is not required, defaults are used. +func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHosts { + return func(host string) ([]docker.RegistryHost, error) { + var hosts []hostConfig + if options.HostDir != nil { + dir, err := options.HostDir(host) + if err != nil && !errdefs.IsNotFound(err) { + return nil, err + } + if dir != "" { + hosts, err = loadHostDir(ctx, dir) + if err != nil { + return nil, err + } + } + + } + + // If hosts was not set, add a default host + // NOTE: Check nil here and not empty, the host may be + // intentionally configured to not have any endpoints + if hosts == nil { + hosts = make([]hostConfig, 1) + } + if len(hosts) > 1 { + if hosts[len(hosts)-1].host == "" { + if host == "docker.io" { + hosts[len(hosts)-1].scheme = "https" + hosts[len(hosts)-1].host = "https://registry-1.docker.io" + } else { + hosts[len(hosts)-1].host = host + if options.DefaultScheme != "" { + hosts[len(hosts)-1].scheme = options.DefaultScheme + } else { + hosts[len(hosts)-1].scheme = "https" + } + } + hosts[len(hosts)-1].path = "/v2" + } + } + + defaultTransport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: options.DefaultTLS, + ExpectContinueTimeout: 5 * time.Second, + } + + client := &http.Client{ + Transport: defaultTransport, + } + + authOpts := []docker.AuthorizerOpt{docker.WithAuthClient(client)} + if options.Credentials != nil { + authOpts = append(authOpts, docker.WithAuthCreds(options.Credentials)) + } + authorizer := docker.NewDockerAuthorizer(authOpts...) + + rhosts := make([]docker.RegistryHost, len(hosts)) + for i, host := range hosts { + + rhosts[i].Scheme = host.scheme + rhosts[i].Host = host.host + rhosts[i].Path = host.path + rhosts[i].Capabilities = host.capabilities + + if host.caCerts != nil || host.clientPairs != nil || host.skipVerify != nil { + var tlsConfig *tls.Config + if options.DefaultTLS != nil { + tlsConfig = options.DefaultTLS.Clone() + } else { + tlsConfig = &tls.Config{} + } + if host.skipVerify != nil { + tlsConfig.InsecureSkipVerify = *host.skipVerify + } + if host.caCerts != nil { + if tlsConfig.RootCAs == nil { + rootPool, err := rootSystemPool() + if err != nil { + return nil, errors.Wrap(err, "unable to initialize cert pool") + } + tlsConfig.RootCAs = rootPool + } + for _, f := range host.caCerts { + data, err := ioutil.ReadFile(f) + if err != nil { + return nil, errors.Wrapf(err, "unable to read CA cert %q", f) + } + if !tlsConfig.RootCAs.AppendCertsFromPEM(data) { + return nil, errors.Errorf("unable to load CA cert %q", f) + } + } + } + + if host.clientPairs != nil { + for _, pair := range host.clientPairs { + certPEMBlock, err := ioutil.ReadFile(pair[0]) + if err != nil { + return nil, errors.Wrapf(err, "unable to read CERT file %q", pair[0]) + } + var keyPEMBlock []byte + if pair[1] != "" { + keyPEMBlock, err = ioutil.ReadFile(pair[1]) + if err != nil { + return nil, errors.Wrapf(err, "unable to read CERT file %q", pair[1]) + } + } else { + // Load key block from same PEM file + keyPEMBlock = certPEMBlock + } + cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + return nil, errors.Wrap(err, "failed to load X509 key pair") + } + + tlsConfig.Certificates = append(tlsConfig.Certificates, cert) + } + } + tr := defaultTransport.Clone() + tr.TLSClientConfig = tlsConfig + c := *client + c.Transport = tr + + rhosts[i].Client = &c + rhosts[i].Authorizer = docker.NewDockerAuthorizer(append(authOpts, docker.WithAuthClient(&c))...) + } else { + rhosts[i].Client = client + rhosts[i].Authorizer = authorizer + } + + } + + return rhosts, nil + } + +} + +// HostDirFromRoot returns a function which finds a host directory +// based at the given root. +func HostDirFromRoot(root string) func(string) (string, error) { + return func(host string) (string, error) { + for _, p := range hostPaths(root, host) { + if _, err := os.Stat(p); err == nil { + return p, nil + } else if !os.IsNotExist(err) { + return "", err + } + } + return "", errdefs.ErrNotFound + } +} + +// hostDirectory converts ":port" to "_port_" in directory names +func hostDirectory(host string) string { + idx := strings.LastIndex(host, ":") + if idx > 0 { + return host[:idx] + "_" + host[idx+1:] + "_" + } + return host +} + +func loadHostDir(ctx context.Context, hostsDir string) ([]hostConfig, error) { + b, err := ioutil.ReadFile(filepath.Join(hostsDir, "hosts.toml")) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + if len(b) == 0 { + // If hosts.toml does not exist, fallback to checking for + // certificate files based on Docker's certificate file + // pattern (".crt", ".cert", ".key" files) + return loadCertFiles(ctx, hostsDir) + } + + hosts, err := parseHostsFile(ctx, hostsDir, b) + if err != nil { + log.G(ctx).WithError(err).Error("failed to decode hosts.toml") + // Fallback to checking certificate files + return loadCertFiles(ctx, hostsDir) + } + + return hosts, nil +} + +type hostFileConfig struct { + // Capabilities determine what operations a host is + // capable of performing. Allowed values + // - pull + // - resolve + // - push + Capabilities []string `toml:"capabilities"` + + // CACert can be a string or an array of strings + CACert toml.Primitive `toml:"ca"` + + // TODO: Make this an array (two key types, one for pairs (multiple files), one for single file?) + Client toml.Primitive `toml:"client"` + + SkipVerify bool `toml:"skip_verify"` + + // API (default: "docker") + // API Version (default: "v2") + // Credentials: helper? name? username? alternate domain? token? +} + +type configFile struct { + // hostConfig holds defaults for all hosts as well as + // for the default server + hostFileConfig + + // Server specifies the default server. When `host` is + // also specified, those hosts are tried first. + Server string `toml:"server"` + + // HostConfigs store the per-host configuration + HostConfigs map[string]hostFileConfig `toml:"host"` +} + +func parseHostsFile(ctx context.Context, baseDir string, b []byte) ([]hostConfig, error) { + var c configFile + md, err := toml.Decode(string(b), &c) + if err != nil { + return nil, err + } + + var orderedHosts []string + for _, key := range md.Keys() { + if len(key) >= 2 { + if key[0] == "host" && (len(orderedHosts) == 0 || orderedHosts[len(orderedHosts)-1] != key[1]) { + orderedHosts = append(orderedHosts, key[1]) + } + } + } + + if c.Server != "" { + c.HostConfigs[c.Server] = c.hostFileConfig + orderedHosts = append(orderedHosts, c.Server) + } else if len(orderedHosts) == 0 { + c.HostConfigs[""] = c.hostFileConfig + orderedHosts = append(orderedHosts, "") + } + hosts := make([]hostConfig, len(orderedHosts)) + for i, server := range orderedHosts { + hostConfig := c.HostConfigs[server] + + if !strings.HasPrefix(server, "http") { + server = "https://" + server + } + u, err := url.Parse(server) + if err != nil { + return nil, errors.Errorf("unable to parse server %v", server) + } + hosts[i].scheme = u.Scheme + hosts[i].host = u.Host + + // TODO: Handle path based on registry protocol + // Define a registry protocol type + // OCI v1 - Always use given path as is + // Docker v2 - Always ensure ends with /v2/ + if len(u.Path) > 0 { + u.Path = path.Clean(u.Path) + if !strings.HasSuffix(u.Path, "/v2") { + u.Path = u.Path + "/v2" + } + } else { + u.Path = "/v2" + } + hosts[i].path = u.Path + + if hosts[i].scheme == "https" { + hosts[i].skipVerify = &hostConfig.SkipVerify + } + + if len(hostConfig.Capabilities) > 0 { + for _, c := range hostConfig.Capabilities { + switch strings.ToLower(c) { + case "pull": + hosts[i].capabilities |= docker.HostCapabilityPull + case "resolve": + hosts[i].capabilities |= docker.HostCapabilityResolve + case "push": + hosts[i].capabilities |= docker.HostCapabilityPush + default: + return nil, errors.Errorf("unknown capability %v", c) + } + } + } else { + hosts[i].capabilities = docker.HostCapabilityPull | docker.HostCapabilityResolve | docker.HostCapabilityPush + } + + baseKey := []string{} + if server != "" { + baseKey = append(baseKey, "host", server) + } + caKey := append(baseKey, "ca") + if md.IsDefined(caKey...) { + switch t := md.Type(caKey...); t { + case "String": + var caCert string + if err := md.PrimitiveDecode(hostConfig.CACert, &caCert); err != nil { + return nil, errors.Wrap(err, "failed to decode \"ca\"") + } + hosts[i].caCerts = []string{makeAbsPath(caCert, baseDir)} + case "Array": + var caCerts []string + if err := md.PrimitiveDecode(hostConfig.CACert, &caCerts); err != nil { + return nil, errors.Wrap(err, "failed to decode \"ca\"") + } + for i, p := range caCerts { + caCerts[i] = makeAbsPath(p, baseDir) + } + + hosts[i].caCerts = caCerts + default: + return nil, errors.Errorf("invalid type %v for \"ca\"", t) + } + } + + clientKey := append(baseKey, "client") + if md.IsDefined(clientKey...) { + switch t := md.Type(clientKey...); t { + case "String": + var clientCert string + if err := md.PrimitiveDecode(hostConfig.Client, &clientCert); err != nil { + return nil, errors.Wrap(err, "failed to decode \"ca\"") + } + hosts[i].clientPairs = [][2]string{{makeAbsPath(clientCert, baseDir), ""}} + case "Array": + var clientCerts []interface{} + if err := md.PrimitiveDecode(hostConfig.Client, &clientCerts); err != nil { + return nil, errors.Wrap(err, "failed to decode \"ca\"") + } + for _, pairs := range clientCerts { + switch p := pairs.(type) { + case string: + hosts[i].clientPairs = append(hosts[i].clientPairs, [2]string{makeAbsPath(p, baseDir), ""}) + case []interface{}: + var pair [2]string + if len(p) > 2 { + return nil, errors.Errorf("invalid pair %v for \"client\"", p) + } + for pi, cp := range p { + s, ok := cp.(string) + if !ok { + return nil, errors.Errorf("invalid type %T for \"client\"", cp) + } + pair[pi] = makeAbsPath(s, baseDir) + } + hosts[i].clientPairs = append(hosts[i].clientPairs, pair) + default: + return nil, errors.Errorf("invalid type %T for \"client\"", p) + } + } + default: + return nil, errors.Errorf("invalid type %v for \"client\"", t) + } + } + } + + return hosts, nil +} + +func makeAbsPath(p string, base string) string { + if filepath.IsAbs(p) { + return p + } + return filepath.Join(base, p) +} + +// loadCertsDir loads certs from certsDir like "/etc/docker/certs.d" . +// Compatible with Docker file layout +// - files ending with ".crt" are treated as CA certificate files +// - files ending with ".cert" are treated as client certificates, and +// files with the same name but ending with ".key" are treated as the +// corresponding private key. +// NOTE: If a ".key" file is missing, this function will just return +// the ".cert", which may contain the private key. If the ".cert" file +// does not contain the private key, the caller should detect and error. +func loadCertFiles(ctx context.Context, certsDir string) ([]hostConfig, error) { + fs, err := ioutil.ReadDir(certsDir) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + hosts := make([]hostConfig, 1) + for _, f := range fs { + if !f.IsDir() { + continue + } + if strings.HasSuffix(f.Name(), ".crt") { + hosts[0].caCerts = append(hosts[0].caCerts, filepath.Join(certsDir, f.Name())) + } + if strings.HasSuffix(f.Name(), ".cert") { + var pair [2]string + certFile := f.Name() + pair[0] = filepath.Join(certsDir, certFile) + // Check if key also exists + keyFile := certFile[:len(certFile)-5] + ".key" + if _, err := os.Stat(keyFile); err == nil { + pair[1] = filepath.Join(certsDir, keyFile) + } else if !os.IsNotExist(err) { + return nil, err + } + hosts[0].clientPairs = append(hosts[0].clientPairs, pair) + } + } + return hosts, nil +} diff --git a/remotes/docker/config/hosts_test.go b/remotes/docker/config/hosts_test.go new file mode 100644 index 000000000..936f25246 --- /dev/null +++ b/remotes/docker/config/hosts_test.go @@ -0,0 +1,205 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "bytes" + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/containerd/containerd/remotes/docker" +) + +func TestParseHostFile(t *testing.T) { + ctx := context.Background() + + const testtoml = ` +server = "https://test-default.registry" +ca = "/etc/path/default" + +[host."https://mirror.registry"] + capabilities = ["pull"] + ca = "/etc/certs/mirror.pem" + +[host."https://mirror-bak.registry/us"] + capabilities = ["pull"] + skip_verify = true + +[host."http://mirror.registry"] + capabilities = ["pull"] + +[host."https://test-1.registry"] + capabilities = ["pull", "resolve", "push"] + ca = ["/etc/certs/test-1-ca.pem", "/etc/certs/special.pem"] + client = [["/etc/certs/client.cert", "/etc/certs/client.key"],["/etc/certs/client.pem", ""]] + +[host."https://test-2.registry"] + client = "/etc/certs/client.pem" + +[host."https://test-3.registry"] + client = ["/etc/certs/client-1.pem", "/etc/certs/client-2.pem"] +` + var tb, fb = true, false + var allCaps = docker.HostCapabilityPull | docker.HostCapabilityResolve | docker.HostCapabilityPush + expected := []hostConfig{ + { + scheme: "https", + host: "mirror.registry", + path: "/v2", + capabilities: docker.HostCapabilityPull, + caCerts: []string{filepath.FromSlash("/etc/certs/mirror.pem")}, + skipVerify: &fb, + }, + { + scheme: "https", + host: "mirror-bak.registry", + path: "/us/v2", + capabilities: docker.HostCapabilityPull, + skipVerify: &tb, + }, + { + scheme: "http", + host: "mirror.registry", + path: "/v2", + capabilities: docker.HostCapabilityPull, + }, + { + scheme: "https", + host: "test-1.registry", + path: "/v2", + capabilities: allCaps, + caCerts: []string{filepath.FromSlash("/etc/certs/test-1-ca.pem"), filepath.FromSlash("/etc/certs/special.pem")}, + clientPairs: [][2]string{ + {filepath.FromSlash("/etc/certs/client.cert"), filepath.FromSlash("/etc/certs/client.key")}, + {filepath.FromSlash("/etc/certs/client.pem"), ""}, + }, + skipVerify: &fb, + }, + { + scheme: "https", + host: "test-2.registry", + path: "/v2", + capabilities: allCaps, + clientPairs: [][2]string{ + {filepath.FromSlash("/etc/certs/client.pem")}, + }, + skipVerify: &fb, + }, + { + scheme: "https", + host: "test-3.registry", + path: "/v2", + capabilities: allCaps, + clientPairs: [][2]string{ + {filepath.FromSlash("/etc/certs/client-1.pem")}, + {filepath.FromSlash("/etc/certs/client-2.pem")}, + }, + skipVerify: &fb, + }, + { + scheme: "https", + host: "test-default.registry", + path: "/v2", + capabilities: allCaps, + skipVerify: &fb, + }, + } + hosts, err := parseHostsFile(ctx, "", []byte(testtoml)) + if err != nil { + t.Fatal(err) + } + + defer func() { + if t.Failed() { + t.Log("HostConfigs...\nActual:\n" + printHostConfig(hosts) + "Expected:\n" + printHostConfig(expected)) + } + }() + + if len(hosts) != len(expected) { + t.Fatalf("Unexpected number of hosts %d, expected %d", len(hosts), len(expected)) + } + + for i := range hosts { + if !compareHostConfig(hosts[i], expected[i]) { + t.Fatalf("Mismatch at host %d", i) + } + } +} + +func compareHostConfig(j, k hostConfig) bool { + if j.scheme != k.scheme { + return false + } + if j.host != k.host { + return false + } + if j.path != k.path { + return false + } + if j.capabilities != k.capabilities { + return false + } + + if len(j.caCerts) != len(k.caCerts) { + return false + } + for i := range j.caCerts { + if j.caCerts[i] != k.caCerts[i] { + return false + } + } + if len(j.clientPairs) != len(k.clientPairs) { + return false + } + for i := range j.clientPairs { + if j.clientPairs[i][0] != k.clientPairs[i][0] { + return false + } + if j.clientPairs[i][1] != k.clientPairs[i][1] { + return false + } + } + if j.skipVerify != nil && k.skipVerify != nil { + if *j.skipVerify != *k.skipVerify { + return false + } + } else if j.skipVerify != nil || k.skipVerify != nil { + return false + } + + return true +} + +func printHostConfig(hc []hostConfig) string { + b := bytes.NewBuffer(nil) + for i := range hc { + fmt.Fprintf(b, "\t[%d]\tscheme: %q\n", i, hc[i].scheme) + fmt.Fprintf(b, "\t\thost: %q\n", hc[i].host) + fmt.Fprintf(b, "\t\tpath: %q\n", hc[i].path) + fmt.Fprintf(b, "\t\tcaps: %03b\n", hc[i].capabilities) + fmt.Fprintf(b, "\t\tca: %#v\n", hc[i].caCerts) + fmt.Fprintf(b, "\t\tclients: %#v\n", hc[i].clientPairs) + if hc[i].skipVerify == nil { + fmt.Fprintf(b, "\t\tskip-verify: %v\n", hc[i].skipVerify) + } else { + fmt.Fprintf(b, "\t\tskip-verify: %t\n", *hc[i].skipVerify) + } + } + return b.String() +} From 547301cb0c674247f0e506912e1ecb3bae23a15f Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 31 Mar 2020 21:46:57 -0700 Subject: [PATCH 3/3] Update ctr resolver to use new config package Moved registry host configuration to the config package and allows support of loading configurations from a directory when the hosts are being resolved. Signed-off-by: Derek McGowan --- cmd/ctr/commands/commands.go | 4 +- cmd/ctr/commands/resolver.go | 80 ++++++---------------- remotes/certutil/certutil.go | 126 ----------------------------------- 3 files changed, 21 insertions(+), 189 deletions(-) delete mode 100644 remotes/certutil/certutil.go diff --git a/cmd/ctr/commands/commands.go b/cmd/ctr/commands/commands.go index 253987730..86202414e 100644 --- a/cmd/ctr/commands/commands.go +++ b/cmd/ctr/commands/commands.go @@ -62,9 +62,9 @@ var ( Usage: "refresh token for authorization server", }, cli.StringFlag{ - Name: "certs-dir", + Name: "hosts-dir", // compatible with "/etc/docker/certs.d" - Usage: "custom certificates directory that contains \"/{ca.crt, client.cert, client.key}\"", + Usage: "Custom hosts configuration directory", }, } diff --git a/cmd/ctr/commands/resolver.go b/cmd/ctr/commands/resolver.go index 5f4b11257..bb394625e 100644 --- a/cmd/ctr/commands/resolver.go +++ b/cmd/ctr/commands/resolver.go @@ -21,20 +21,13 @@ import ( gocontext "context" "crypto/tls" "fmt" - "io/ioutil" - "net" - "net/http" - "os" - "path/filepath" "strings" - "time" "github.com/containerd/console" "github.com/containerd/containerd/remotes" - "github.com/containerd/containerd/remotes/certutil" "github.com/containerd/containerd/remotes/docker" + "github.com/containerd/containerd/remotes/docker/config" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -65,8 +58,7 @@ func GetResolver(ctx gocontext.Context, clicontext *cli.Context) (remotes.Resolv username = username[0:i] } options := docker.ResolverOptions{ - PlainHTTP: clicontext.Bool("plain-http"), - Tracker: PushTracker, + Tracker: PushTracker, } if username != "" { if secret == "" { @@ -84,60 +76,26 @@ func GetResolver(ctx gocontext.Context, clicontext *cli.Context) (remotes.Resolv secret = rt } - tr := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 10, - IdleConnTimeout: 30 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: clicontext.Bool("skip-verify"), - }, - ExpectContinueTimeout: 5 * time.Second, - } - loadCertsDir(tr.TLSClientConfig, clicontext.String("certs-dir")) - - options.Client = &http.Client{ - Transport: tr, - } - - credentials := func(host string) (string, string, error) { + hostOptions := config.HostOptions{} + hostOptions.Credentials = func(host string) (string, string, error) { + // If host doesn't match... // Only one host return username, secret, nil } - authOpts := []docker.AuthorizerOpt{docker.WithAuthClient(options.Client), docker.WithAuthCreds(credentials)} - options.Authorizer = docker.NewDockerAuthorizer(authOpts...) + if clicontext.Bool("plain-http") { + hostOptions.DefaultScheme = "http" + } + + if clicontext.Bool("skip-verify") { + hostOptions.DefaultTLS = &tls.Config{ + InsecureSkipVerify: true, + } + } + if hostDir := clicontext.String("hosts-dir"); hostDir != "" { + hostOptions.HostDir = config.HostDirFromRoot(hostDir) + } + + options.Hosts = config.ConfigureHosts(ctx, hostOptions) return docker.NewResolver(options), nil } - -// loadCertsDir loads certs from certsDir like "/etc/docker/certs.d" . -func loadCertsDir(config *tls.Config, certsDir string) { - if certsDir == "" { - return - } - fs, err := ioutil.ReadDir(certsDir) - if err != nil && !os.IsNotExist(err) { - logrus.WithError(err).Errorf("cannot read certs directory %q", certsDir) - return - } - for _, f := range fs { - if !f.IsDir() { - continue - } - // TODO: skip loading if f.Name() is not valid FQDN/IP - hostDir := filepath.Join(certsDir, f.Name()) - caCertGlob := filepath.Join(hostDir, "*.crt") - if _, err = certutil.LoadCACerts(config, caCertGlob); err != nil { - logrus.WithError(err).Errorf("cannot load certs from %q", caCertGlob) - } - keyGlob := filepath.Join(hostDir, "*.key") - if _, _, err = certutil.LoadKeyPairs(config, keyGlob, certutil.DockerKeyPairCertLocator); err != nil { - logrus.WithError(err).Errorf("cannot load key pairs from %q", keyGlob) - } - } -} diff --git a/remotes/certutil/certutil.go b/remotes/certutil/certutil.go deleted file mode 100644 index 7755388b3..000000000 --- a/remotes/certutil/certutil.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certutil - -import ( - "crypto/tls" - "crypto/x509" - "io/ioutil" - "path/filepath" - "runtime" - "sort" - "strings" - - "github.com/containerd/containerd/errdefs" - "github.com/pkg/errors" -) - -// SystemCertPool returns a copy of the system cert pool, -// returns an error if failed to load or empty pool on windows. -// -// SystemCertPool was ported from Docker 19.03 -// https://github.com/docker/engine/blob/v19.03.1/vendor/github.com/docker/go-connections/tlsconfig/certpool_go17.go#L12 -func SystemCertPool() (*x509.CertPool, error) { - certpool, err := x509.SystemCertPool() - if err != nil && runtime.GOOS == "windows" { - return x509.NewCertPool(), nil - } - return certpool, err -} - -// LoadCACerts loads CA certificates into tlsConfig from glob. -// glob should be like "/etc/docker/certs.d/example.com/*.crt" . -// LoadCACerts returns the paths of the loaded certs. -func LoadCACerts(tlsConfig *tls.Config, glob string) ([]string, error) { - if tlsConfig == nil { - tlsConfig = &tls.Config{} - } - files, err := filepath.Glob(glob) - if err != nil { - return nil, err - } - sort.Strings(files) - if tlsConfig.RootCAs == nil { - systemPool, err := SystemCertPool() - if err != nil { - return nil, errors.Wrap(err, "unable to get system cert poolv") - } - tlsConfig.RootCAs = systemPool - } - var loaded []string - for _, f := range files { - data, err := ioutil.ReadFile(f) - if err != nil { - return loaded, errors.Wrapf(err, "unable to read CA cert %q", f) - } - if !tlsConfig.RootCAs.AppendCertsFromPEM(data) { - return loaded, errors.Errorf("unable to load CA cert %q", f) - } - loaded = append(loaded, f) - } - return loaded, nil -} - -// KeyPairCertLocator is used to resolve the cert path from the key path. -type KeyPairCertLocator func(keyPath string) (certPath string, err error) - -// DockerKeyPairCertLocator implements the Docker-style convention. ("*.key" -> "*.cert") -var DockerKeyPairCertLocator KeyPairCertLocator = func(keyPath string) (string, error) { - if !strings.HasSuffix(keyPath, ".key") { - return "", errors.Errorf("expected key path with \".key\" suffix, got %q", keyPath) - } - // Docker convention uses *.crt for CA certs, *.cert for keypair certs. - certPath := keyPath[:len(keyPath)-4] + ".cert" - return certPath, nil -} - -// LoadKeyPairs loads key pairs into tlsConfig from keyGlob. -// keyGlob should be like "/etc/docker/certs.d/example.com/*.key" . -// certLocator is used to resolve the cert path from the key path. -// Use DockerKeyPairCertLocator for the Docker-style convention. ("*.key" -> "*.cert") -// LoadKeyParis returns the paths of the loaded certs and the loaded keys. -func LoadKeyPairs(tlsConfig *tls.Config, keyGlob string, certLocator KeyPairCertLocator) ([]string, []string, error) { - if tlsConfig == nil { - tlsConfig = &tls.Config{} - } - if certLocator == nil { - return nil, nil, errors.Wrap(errdefs.ErrInvalidArgument, "missing cert locator") - } - keyPaths, err := filepath.Glob(keyGlob) - if err != nil { - return nil, nil, err - } - sort.Strings(keyPaths) - var ( - loadedCerts []string - loadedKeys []string - ) - for _, keyPath := range keyPaths { - certPath, err := certLocator(keyPath) - if err != nil { - return loadedCerts, loadedKeys, err - } - keyPair, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { - return loadedCerts, loadedKeys, err - } - tlsConfig.Certificates = append(tlsConfig.Certificates, keyPair) - loadedCerts = append(loadedCerts, certPath) - loadedKeys = append(loadedKeys, keyPath) - } - return loadedCerts, loadedKeys, nil -}