Compute manifest metadata when not provided.

This closes #3238

Signed-off-by: msg555 <msg555@gmail.com>

remotes/docker/resolver.go

@@ -18,10 +18,10 @@ package docker
 
 import (
 	"context"
+	"io"
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 
 	"github.com/containerd/containerd/errdefs"
@@ -29,6 +29,7 @@ import (
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker/schema1"
 	"github.com/containerd/containerd/version"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -150,6 +151,32 @@ func NewResolver(options ResolverOptions) remotes.Resolver {
 	}
 }
 
+func getManifestMediaType(resp *http.Response) string {
+	// Strip encoding data (manifests should always be ascii JSON)
+	contentType := resp.Header.Get("Content-Type")
+	if sp := strings.IndexByte(contentType, ';'); sp != -1 {
+		contentType = contentType[0:sp]
+	}
+
+	// As of Apr 30 2019 the registry.access.redhat.com registry does not specify
+	// the content type of any data but uses schema1 manifests.
+	if contentType == "text/plain" {
+		contentType = images.MediaTypeDockerSchema1Manifest
+	}
+	return contentType
+}
+
+type countingReader struct {
+	reader    io.Reader
+	bytesRead int64
+}
+
+func (r *countingReader) Read(p []byte) (int, error) {
+	n, err := r.reader.Read(p)
+	r.bytesRead += int64(n)
+	return n, err
+}
+
 var _ remotes.Resolver = &dockerResolver{}
 
 func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) {
@@ -220,40 +247,56 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
 			}
 			return "", ocispec.Descriptor{}, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
 		}
+		size := resp.ContentLength
 
 		// this is the only point at which we trust the registry. we use the
 		// content headers to assemble a descriptor for the name. when this becomes
 		// more robust, we mostly get this information from a secure trust store.
 		dgstHeader := digest.Digest(resp.Header.Get("Docker-Content-Digest"))
+		contentType := getManifestMediaType(resp)
 
-		if dgstHeader != "" {
+		if dgstHeader != "" && size != -1 {
 			if err := dgstHeader.Validate(); err != nil {
 				return "", ocispec.Descriptor{}, errors.Wrapf(err, "%q in header not a valid digest", dgstHeader)
 			}
 			dgst = dgstHeader
-		}
+		} else {
+			log.G(ctx).Debug("no Docker-Content-Digest header, fetching manifest instead")
 
-		if dgst == "" {
+			req, err := http.NewRequest(http.MethodGet, u, nil)
-			return "", ocispec.Descriptor{}, errors.Errorf("could not resolve digest for %v", ref)
+			if err != nil {
-		}
+				return "", ocispec.Descriptor{}, err
+			}
+			req.Header = r.headers
 
-		var (
+			resp, err := fetcher.doRequestWithRetries(ctx, req, nil)
-			size       int64
+			if err != nil {
-			sizeHeader = resp.Header.Get("Content-Length")
+				return "", ocispec.Descriptor{}, err
-		)
+			}
+			defer resp.Body.Close()
 
-		size, err = strconv.ParseInt(sizeHeader, 10, 64)
+			bodyReader := countingReader{reader: resp.Body}
-		if err != nil {
 
-			return "", ocispec.Descriptor{}, errors.Wrapf(err, "invalid size header: %q", sizeHeader)
+			contentType = getManifestMediaType(resp)
-		}
+			if contentType == images.MediaTypeDockerSchema1Manifest {
-		if size < 0 {
+				b, err := schema1.ReadStripSignature(&bodyReader)
-			return "", ocispec.Descriptor{}, errors.Errorf("%q in header not a valid size", sizeHeader)
+				if err != nil {
+					return "", ocispec.Descriptor{}, err
+				}
+
+				dgst = digest.FromBytes(b)
+			} else {
+				dgst, err = digest.FromReader(&bodyReader)
+				if err != nil {
+					return "", ocispec.Descriptor{}, err
+				}
+			}
+			size = bodyReader.bytesRead
 		}
 
 		desc := ocispec.Descriptor{
 			Digest:    dgst,
-			MediaType: resp.Header.Get("Content-Type"), // need to strip disposition?
+			MediaType: contentType,
 			Size:      size,
 		}
 

remotes/docker/schema1/converter.go

@@ -227,6 +227,17 @@ func (c *Converter) Convert(ctx context.Context, opts ...ConvertOpt) (ocispec.De
 	return desc, nil
 }
 
+// ReadStripSignature reads in a schema1 manifest and returns a byte array
+// with the "signatures" field stripped
+func ReadStripSignature(schema1Blob io.Reader) ([]byte, error) {
+	b, err := ioutil.ReadAll(io.LimitReader(schema1Blob, manifestSizeLimit)) // limit to 8MB
+	if err != nil {
+		return nil, err
+	}
+
+	return stripSignature(b)
+}
+
 func (c *Converter) fetchManifest(ctx context.Context, desc ocispec.Descriptor) error {
 	log.G(ctx).Debug("fetch schema 1")
 
@@ -235,17 +246,12 @@ func (c *Converter) fetchManifest(ctx context.Context, desc ocispec.Descriptor)
 		return err
 	}
 
-	b, err := ioutil.ReadAll(io.LimitReader(rc, manifestSizeLimit)) // limit to 8MB
+	b, err := ReadStripSignature(rc)
 	rc.Close()
 	if err != nil {
 		return err
 	}
 
-	b, err = stripSignature(b)
-	if err != nil {
-		return err
-	}
-
 	var m manifest
 	if err := json.Unmarshal(b, &m); err != nil {
 		return err