From 06b0cd45ba7b40a80e239d66827f421c674f6e49 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 19 May 2020 18:50:27 -0700 Subject: [PATCH 1/3] Fix nil pointer errors Signed-off-by: Derek McGowan --- remotes/docker/config/hosts.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/remotes/docker/config/hosts.go b/remotes/docker/config/hosts.go index ae984219d..ce59428ad 100644 --- a/remotes/docker/config/hosts.go +++ b/remotes/docker/config/hosts.go @@ -105,6 +105,13 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos hosts[len(hosts)-1].capabilities = docker.HostCapabilityPull | docker.HostCapabilityResolve | docker.HostCapabilityPush } + var defaultTLSConfig *tls.Config + if options.DefaultTLS != nil { + defaultTLSConfig = options.DefaultTLS + } else { + defaultTLSConfig = &tls.Config{} + } + defaultTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ @@ -115,7 +122,7 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, TLSHandshakeTimeout: 10 * time.Second, - TLSClientConfig: options.DefaultTLS, + TLSClientConfig: defaultTLSConfig, ExpectContinueTimeout: 5 * time.Second, } @@ -138,12 +145,8 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos rhosts[i].Capabilities = host.capabilities if host.caCerts != nil || host.clientPairs != nil || host.skipVerify != nil { - var tlsConfig *tls.Config - if options.DefaultTLS != nil { - tlsConfig = options.DefaultTLS.Clone() - } else { - tlsConfig = &tls.Config{} - } + tr := defaultTransport.Clone() + tlsConfig := tr.TLSClientConfig if host.skipVerify != nil { tlsConfig.InsecureSkipVerify = *host.skipVerify } @@ -190,8 +193,7 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos tlsConfig.Certificates = append(tlsConfig.Certificates, cert) } } - tr := defaultTransport.Clone() - tr.TLSClientConfig = tlsConfig + c := *client c.Transport = tr @@ -306,6 +308,9 @@ func parseHostsFile(ctx context.Context, baseDir string, b []byte) ([]hostConfig } } + if c.HostConfigs == nil { + c.HostConfigs = map[string]hostFileConfig{} + } if c.Server != "" { c.HostConfigs[c.Server] = c.hostFileConfig orderedHosts = append(orderedHosts, c.Server) From 84619ee99812fa865e19da85ffebcfaf890bbcb2 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 19 May 2020 18:31:07 -0700 Subject: [PATCH 2/3] Fix configurations with no server provided When a server is specified at the top level, there is a bug that prevents the keys from being checked properly. When no server is provided, the server attempts to parse with an empty host, leaving partial values and a defaulted skip verify configuration. Signed-off-by: Derek McGowan --- remotes/docker/config/hosts.go | 51 ++++++++++++++--------------- remotes/docker/config/hosts_test.go | 6 ++-- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/remotes/docker/config/hosts.go b/remotes/docker/config/hosts.go index ce59428ad..100f6be1f 100644 --- a/remotes/docker/config/hosts.go +++ b/remotes/docker/config/hosts.go @@ -272,7 +272,7 @@ type hostFileConfig struct { // TODO: Make this an array (two key types, one for pairs (multiple files), one for single file?) Client toml.Primitive `toml:"client"` - SkipVerify bool `toml:"skip_verify"` + SkipVerify *bool `toml:"skip_verify"` // API (default: "docker") // API Version (default: "v2") @@ -322,33 +322,32 @@ func parseHostsFile(ctx context.Context, baseDir string, b []byte) ([]hostConfig for i, server := range orderedHosts { hostConfig := c.HostConfigs[server] - if !strings.HasPrefix(server, "http") { - server = "https://" + server - } - u, err := url.Parse(server) - if err != nil { - return nil, errors.Errorf("unable to parse server %v", server) - } - hosts[i].scheme = u.Scheme - hosts[i].host = u.Host - - // TODO: Handle path based on registry protocol - // Define a registry protocol type - // OCI v1 - Always use given path as is - // Docker v2 - Always ensure ends with /v2/ - if len(u.Path) > 0 { - u.Path = path.Clean(u.Path) - if !strings.HasSuffix(u.Path, "/v2") { - u.Path = u.Path + "/v2" + if server != "" { + if !strings.HasPrefix(server, "http") { + server = "https://" + server } - } else { - u.Path = "/v2" - } - hosts[i].path = u.Path + u, err := url.Parse(server) + if err != nil { + return nil, errors.Errorf("unable to parse server %v", server) + } + hosts[i].scheme = u.Scheme + hosts[i].host = u.Host - if hosts[i].scheme == "https" { - hosts[i].skipVerify = &hostConfig.SkipVerify + // TODO: Handle path based on registry protocol + // Define a registry protocol type + // OCI v1 - Always use given path as is + // Docker v2 - Always ensure ends with /v2/ + if len(u.Path) > 0 { + u.Path = path.Clean(u.Path) + if !strings.HasSuffix(u.Path, "/v2") { + u.Path = u.Path + "/v2" + } + } else { + u.Path = "/v2" + } + hosts[i].path = u.Path } + hosts[i].skipVerify = hostConfig.SkipVerify if len(hostConfig.Capabilities) > 0 { for _, c := range hostConfig.Capabilities { @@ -368,7 +367,7 @@ func parseHostsFile(ctx context.Context, baseDir string, b []byte) ([]hostConfig } baseKey := []string{} - if server != "" { + if server != "" && server != c.Server { baseKey = append(baseKey, "host", server) } caKey := append(baseKey, "ca") diff --git a/remotes/docker/config/hosts_test.go b/remotes/docker/config/hosts_test.go index 35cab475a..18857a000 100644 --- a/remotes/docker/config/hosts_test.go +++ b/remotes/docker/config/hosts_test.go @@ -80,6 +80,7 @@ ca = "/etc/path/default" [host."https://mirror.registry"] capabilities = ["pull"] ca = "/etc/certs/mirror.pem" + skip_verify = false [host."https://mirror-bak.registry/us"] capabilities = ["pull"] @@ -132,7 +133,6 @@ ca = "/etc/path/default" {filepath.FromSlash("/etc/certs/client.cert"), filepath.FromSlash("/etc/certs/client.key")}, {filepath.FromSlash("/etc/certs/client.pem"), ""}, }, - skipVerify: &fb, }, { scheme: "https", @@ -142,7 +142,6 @@ ca = "/etc/path/default" clientPairs: [][2]string{ {filepath.FromSlash("/etc/certs/client.pem")}, }, - skipVerify: &fb, }, { scheme: "https", @@ -153,14 +152,13 @@ ca = "/etc/path/default" {filepath.FromSlash("/etc/certs/client-1.pem")}, {filepath.FromSlash("/etc/certs/client-2.pem")}, }, - skipVerify: &fb, }, { scheme: "https", host: "test-default.registry", path: "/v2", capabilities: allCaps, - skipVerify: &fb, + caCerts: []string{filepath.FromSlash("/etc/path/default")}, }, } hosts, err := parseHostsFile(ctx, "", []byte(testtoml)) From 3dd8242a67a6ee9bff4cd22db5e2b95d1231dbf3 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 18 May 2020 16:38:45 -0700 Subject: [PATCH 3/3] Add host specific headers Allows configuring headers per registry host Signed-off-by: Derek McGowan --- remotes/docker/config/hosts.go | 31 ++++++++++++++++++++++++++++- remotes/docker/config/hosts_test.go | 22 ++++++++++++++++++++ remotes/docker/registry.go | 1 + remotes/docker/resolver.go | 3 +++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/remotes/docker/config/hosts.go b/remotes/docker/config/hosts.go index 100f6be1f..60bf9148e 100644 --- a/remotes/docker/config/hosts.go +++ b/remotes/docker/config/hosts.go @@ -48,6 +48,8 @@ type hostConfig struct { clientPairs [][2]string skipVerify *bool + header http.Header + // TODO: API ("docker" or "oci") // TODO: API Version ("v1", "v2") // TODO: Add credential configuration (domain alias, username) @@ -143,6 +145,7 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos rhosts[i].Host = host.host rhosts[i].Path = host.path rhosts[i].Capabilities = host.capabilities + rhosts[i].Header = host.header if host.caCerts != nil || host.clientPairs != nil || host.skipVerify != nil { tr := defaultTransport.Clone() @@ -203,7 +206,6 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos rhosts[i].Client = client rhosts[i].Authorizer = authorizer } - } return rhosts, nil @@ -274,6 +276,8 @@ type hostFileConfig struct { SkipVerify *bool `toml:"skip_verify"` + Header map[string]toml.Primitive `toml:"header"` + // API (default: "docker") // API Version (default: "v2") // Credentials: helper? name? username? alternate domain? token? @@ -433,6 +437,31 @@ func parseHostsFile(ctx context.Context, baseDir string, b []byte) ([]hostConfig return nil, errors.Errorf("invalid type %v for \"client\"", t) } } + + headerKey := append(baseKey, "header") + if md.IsDefined(headerKey...) { + header := http.Header{} + for key, prim := range hostConfig.Header { + switch t := md.Type(append(headerKey, key)...); t { + case "String": + var value string + if err := md.PrimitiveDecode(prim, &value); err != nil { + return nil, errors.Wrapf(err, "failed to decode header %q", key) + } + header[key] = []string{value} + case "Array": + var value []string + if err := md.PrimitiveDecode(prim, &value); err != nil { + return nil, errors.Wrapf(err, "failed to decode header %q", key) + } + + header[key] = value + default: + return nil, errors.Errorf("invalid type %v for header %q", t, key) + } + } + hosts[i].header = header + } } return hosts, nil diff --git a/remotes/docker/config/hosts_test.go b/remotes/docker/config/hosts_test.go index 18857a000..194d3d8af 100644 --- a/remotes/docker/config/hosts_test.go +++ b/remotes/docker/config/hosts_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "net/http" "path/filepath" "testing" @@ -76,11 +77,15 @@ func TestParseHostFile(t *testing.T) { const testtoml = ` server = "https://test-default.registry" ca = "/etc/path/default" +[header] + x-custom-1 = "custom header" [host."https://mirror.registry"] capabilities = ["pull"] ca = "/etc/certs/mirror.pem" skip_verify = false + [host."https://mirror.registry".header] + x-custom-2 = ["value1", "value2"] [host."https://mirror-bak.registry/us"] capabilities = ["pull"] @@ -109,6 +114,7 @@ ca = "/etc/path/default" capabilities: docker.HostCapabilityPull, caCerts: []string{filepath.FromSlash("/etc/certs/mirror.pem")}, skipVerify: &fb, + header: http.Header{"x-custom-2": {"value1", "value2"}}, }, { scheme: "https", @@ -159,6 +165,7 @@ ca = "/etc/path/default" path: "/v2", capabilities: allCaps, caCerts: []string{filepath.FromSlash("/etc/path/default")}, + header: http.Header{"x-custom-1": {"custom header"}}, }, } hosts, err := parseHostsFile(ctx, "", []byte(testtoml)) @@ -241,6 +248,20 @@ func compareHostConfig(j, k hostConfig) bool { return false } + if len(j.header) != len(k.header) { + return false + } + for key := range j.header { + if len(j.header[key]) != len(k.header[key]) { + return false + } + for i := range j.header[key] { + if j.header[key][i] != k.header[key][i] { + return false + } + } + } + return true } @@ -258,6 +279,7 @@ func printHostConfig(hc []hostConfig) string { } else { fmt.Fprintf(b, "\t\tskip-verify: %t\n", *hc[i].skipVerify) } + fmt.Fprintf(b, "\t\theader: %#v\n", hc[i].header) } return b.String() } diff --git a/remotes/docker/registry.go b/remotes/docker/registry.go index ae24f41e1..ffc939b40 100644 --- a/remotes/docker/registry.go +++ b/remotes/docker/registry.go @@ -70,6 +70,7 @@ type RegistryHost struct { Scheme string Path string Capabilities HostCapabilities + Header http.Header } // RegistryHosts fetches the registry hosts for a given namespace, diff --git a/remotes/docker/resolver.go b/remotes/docker/resolver.go index 503f72aaf..32b6abd90 100644 --- a/remotes/docker/resolver.go +++ b/remotes/docker/resolver.go @@ -450,6 +450,9 @@ func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *re for key, value := range r.header { header[key] = append(header[key], value...) } + for key, value := range host.Header { + header[key] = append(header[key], value...) + } parts := append([]string{"/", host.Path, r.namespace}, ps...) p := path.Join(parts...) // Join strips trailing slash, re-add ending "/" if included