Support HTTP debug in ctr

Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>

cmd/ctr/commands/commands.go

@@ -78,6 +78,14 @@ var (
 			Name:  "tlskey",
 			Usage: "path to TLS client key",
 		},
+		cli.BoolFlag{
+			Name:  "http-dump",
+			Usage: "dump all HTTP request/responses when interacting with container registry",
+		},
+		cli.BoolFlag{
+			Name:  "http-trace",
+			Usage: "enable HTTP tracing for registry interactions",
+		},
 	}
 
 	// ContainerFlags are cli flags specifying container options

cmd/ctr/commands/content/fetch.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http/httptrace"
 	"os"
 	"sync"
 	"text/tabwriter"
@@ -110,6 +111,8 @@ type FetchConfig struct {
 	AllMetadata bool
 	// RemoteOpts is not used by ctr, but can be used by other CLI tools
 	RemoteOpts []containerd.RemoteOpt
+	// TraceHTTP writes DNS and connection information to the log when dealing with a container registry
+	TraceHTTP bool
 }
 
 // NewFetchConfig returns the default FetchConfig from cli flags
@@ -121,6 +124,7 @@ func NewFetchConfig(ctx context.Context, clicontext *cli.Context) (*FetchConfig,
 	config := &FetchConfig{
 		Resolver:  resolver,
 		Labels:    clicontext.StringSlice("label"),
+		TraceHTTP: clicontext.Bool("http-trace"),
 	}
 	if !clicontext.GlobalBool("debug") {
 		config.ProgressOutput = os.Stdout
@@ -148,6 +152,10 @@ func NewFetchConfig(ctx context.Context, clicontext *cli.Context) (*FetchConfig,
 func Fetch(ctx context.Context, client *containerd.Client, ref string, config *FetchConfig) (images.Image, error) {
 	ongoing := NewJobs(ref)
 
+	if config.TraceHTTP {
+		ctx = httptrace.WithClientTrace(ctx, commands.NewDebugClientTrace(ctx))
+	}
+
 	pctx, stopProgress := context.WithCancel(ctx)
 	progress := make(chan struct{})
 

cmd/ctr/commands/images/images.go

@@ -17,9 +17,7 @@
 package images
 
 import (
-	"context"
 	"fmt"
-	"net/http/httptrace"
 	"os"
 	"sort"
 	"strings"
@@ -334,23 +332,3 @@ var removeCommand = cli.Command{
 		return exitErr
 	},
 }
-
-// NewDebugClientTrace returns a Go http trace client predefined to write DNS and connection
-// information to the log. This is used via the --trace flag on push and pull operations in ctr.
-func NewDebugClientTrace(ctx context.Context) *httptrace.ClientTrace {
-	return &httptrace.ClientTrace{
-		DNSStart: func(dnsInfo httptrace.DNSStartInfo) {
-			log.G(ctx).WithField("host", dnsInfo.Host).Debugf("DNS lookup")
-		},
-		DNSDone: func(dnsInfo httptrace.DNSDoneInfo) {
-			if dnsInfo.Err != nil {
-				log.G(ctx).WithField("lookup_err", dnsInfo.Err).Debugf("DNS lookup error")
-			} else {
-				log.G(ctx).WithField("result", dnsInfo.Addrs[0].String()).WithField("coalesced", dnsInfo.Coalesced).Debugf("DNS lookup complete")
-			}
-		},
-		GotConn: func(connInfo httptrace.GotConnInfo) {
-			log.G(ctx).WithField("reused", connInfo.Reused).WithField("remote_addr", connInfo.Conn.RemoteAddr().String()).Debugf("Connection successful")
-		},
-	}
-}

cmd/ctr/commands/images/pull.go

@@ -18,7 +18,6 @@ package images
 
 import (
 	"fmt"
-	"net/http/httptrace"
 	"time"
 
 	"github.com/containerd/containerd"
@@ -56,10 +55,6 @@ command. As part of this process, we do the following:
 			Name:  "all-platforms",
 			Usage: "pull content and metadata from all platforms",
 		},
-		cli.BoolFlag{
-			Name:  "trace",
-			Usage: "enable HTTP tracing for registry interactions",
-		},
 		cli.BoolFlag{
 			Name:  "all-metadata",
 			Usage: "Pull metadata for all platforms",
@@ -94,9 +89,6 @@ command. As part of this process, we do the following:
 			return err
 		}
 
-		if context.Bool("trace") {
-			ctx = httptrace.WithClientTrace(ctx, NewDebugClientTrace(ctx))
-		}
 		img, err := content.Fetch(ctx, client, ref, config)
 		if err != nil {
 			return err

cmd/ctr/commands/images/push.go

@@ -60,9 +60,6 @@ var pushCommand = cli.Command{
 		Name:  "manifest-type",
 		Usage: "media type of manifest digest",
 		Value: ocispec.MediaTypeImageManifest,
-	}, cli.BoolFlag{
-		Name:  "trace",
-		Usage: "enable HTTP tracing for registry interactions",
 	}, cli.StringSliceFlag{
 		Name:  "platform",
 		Usage: "push content from a specific platform",
@@ -123,8 +120,8 @@ var pushCommand = cli.Command{
 			}
 		}
 
-		if context.Bool("trace") {
+		if context.Bool("http-trace") {
-			ctx = httptrace.WithClientTrace(ctx, NewDebugClientTrace(ctx))
+			ctx = httptrace.WithClientTrace(ctx, commands.NewDebugClientTrace(ctx))
 		}
 		resolver, err := commands.GetResolver(ctx, context)
 		if err != nil {

cmd/ctr/commands/resolver.go

@@ -22,10 +22,15 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
+	"net/http/httptrace"
+	"net/http/httputil"
 	"strings"
 
 	"github.com/containerd/console"
+	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/remotes/docker/config"
@@ -96,6 +101,16 @@ func GetResolver(ctx gocontext.Context, clicontext *cli.Context) (remotes.Resolv
 		hostOptions.HostDir = config.HostDirFromRoot(hostDir)
 	}
 
+	if clicontext.Bool("http-dump") {
+		hostOptions.UpdateClient = func(client *http.Client) error {
+			client.Transport = &DebugTransport{
+				transport: client.Transport,
+				writer:    log.G(ctx).Writer(),
+			}
+			return nil
+		}
+	}
+
 	options.Hosts = config.ConfigureHosts(ctx, hostOptions)
 
 	return docker.NewResolver(options), nil
@@ -135,3 +150,57 @@ func resolverDefaultTLS(clicontext *cli.Context) (*tls.Config, error) {
 
 	return config, nil
 }
+
+// DebugTransport wraps the underlying http.RoundTripper interface and dumps all requests/responses to the writer.
+type DebugTransport struct {
+	transport http.RoundTripper
+	writer    io.Writer
+}
+
+// RoundTrip dumps request/responses and executes the request using the underlying transport.
+func (t DebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	in, err := httputil.DumpRequest(req, true)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to dump request")
+	}
+
+	if _, err := t.writer.Write(in); err != nil {
+		return nil, err
+	}
+
+	resp, err := t.transport.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := httputil.DumpResponse(resp, true)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to dump response")
+	}
+
+	if _, err := t.writer.Write(out); err != nil {
+		return nil, err
+	}
+
+	return resp, err
+}
+
+// NewDebugClientTrace returns a Go http trace client predefined to write DNS and connection
+// information to the log. This is used via the --http-trace flag on push and pull operations in ctr.
+func NewDebugClientTrace(ctx gocontext.Context) *httptrace.ClientTrace {
+	return &httptrace.ClientTrace{
+		DNSStart: func(dnsInfo httptrace.DNSStartInfo) {
+			log.G(ctx).WithField("host", dnsInfo.Host).Debugf("DNS lookup")
+		},
+		DNSDone: func(dnsInfo httptrace.DNSDoneInfo) {
+			if dnsInfo.Err != nil {
+				log.G(ctx).WithField("lookup_err", dnsInfo.Err).Debugf("DNS lookup error")
+			} else {
+				log.G(ctx).WithField("result", dnsInfo.Addrs[0].String()).WithField("coalesced", dnsInfo.Coalesced).Debugf("DNS lookup complete")
+			}
+		},
+		GotConn: func(connInfo httptrace.GotConnInfo) {
+			log.G(ctx).WithField("reused", connInfo.Reused).WithField("remote_addr", connInfo.Conn.RemoteAddr().String()).Debugf("Connection successful")
+		},
+	}
+}

remotes/docker/config/hosts.go

@@ -37,6 +37,9 @@ import (
 	"github.com/pkg/errors"
 )
 
+// UpdateClientFunc is a function that lets you to amend http Client behavior used by registry clients.
+type UpdateClientFunc func(client *http.Client) error
+
 type hostConfig struct {
 	scheme string
 	host   string
@@ -61,6 +64,8 @@ type HostOptions struct {
 	Credentials   func(host string) (string, string, error)
 	DefaultTLS    *tls.Config
 	DefaultScheme string
+	// UpdateClient will be called after creating http.Client object, so clients can provide extra configuration
+	UpdateClient UpdateClientFunc
 }
 
 // ConfigureHosts creates a registry hosts function from the provided
@@ -130,6 +135,11 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos
 		client := &http.Client{
 			Transport: defaultTransport,
 		}
+		if options.UpdateClient != nil {
+			if err := options.UpdateClient(client); err != nil {
+				return nil, err
+			}
+		}
 
 		authOpts := []docker.AuthorizerOpt{docker.WithAuthClient(client)}
 		if options.Credentials != nil {
@@ -198,6 +208,11 @@ func ConfigureHosts(ctx context.Context, options HostOptions) docker.RegistryHos
 
 				c := *client
 				c.Transport = tr
+				if options.UpdateClient != nil {
+					if err := options.UpdateClient(&c); err != nil {
+						return nil, err
+					}
+				}
 
 				rhosts[i].Client = &c
 				rhosts[i].Authorizer = docker.NewDockerAuthorizer(append(authOpts, docker.WithAuthClient(&c))...)