images, remotes: add support for foreign layers

Add support for downloading layers with external URLs and
foreign/non-distributable mediatypes. This ensures that encountered
windows images are downloaded correctly. We still need to filter out the
extra windows resources when pulling linux, but this is a step towards
correctly supporting multi-platform images.

Signed-off-by: Stephen J Day <stephen.day@docker.com>

images/handlers.go

@@ -159,8 +159,10 @@ func ChildrenHandler(provider content.Provider) HandlerFunc {
 
 			descs = append(descs, index.Manifests...)
 		case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
+			MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip,
 			MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
-			ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip:
+			ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
+			ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip:
 			// childless data types.
 			return nil, nil
 		default:

images/mediatypes.go

@@ -5,11 +5,13 @@ package images
 // oci components are generally referenced directly, although we may centralize
 // here for clarity.
 const (
-	MediaTypeDockerSchema2Layer        = "application/vnd.docker.image.rootfs.diff.tar"
+	MediaTypeDockerSchema2Layer            = "application/vnd.docker.image.rootfs.diff.tar"
-	MediaTypeDockerSchema2LayerGzip    = "application/vnd.docker.image.rootfs.diff.tar.gzip"
+	MediaTypeDockerSchema2LayerForeign     = "application/vnd.docker.image.rootfs.foreign.diff.tar"
-	MediaTypeDockerSchema2Config       = "application/vnd.docker.container.image.v1+json"
+	MediaTypeDockerSchema2LayerGzip        = "application/vnd.docker.image.rootfs.diff.tar.gzip"
-	MediaTypeDockerSchema2Manifest     = "application/vnd.docker.distribution.manifest.v2+json"
+	MediaTypeDockerSchema2LayerForeignGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
-	MediaTypeDockerSchema2ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
+	MediaTypeDockerSchema2Config           = "application/vnd.docker.container.image.v1+json"
+	MediaTypeDockerSchema2Manifest         = "application/vnd.docker.distribution.manifest.v2+json"
+	MediaTypeDockerSchema2ManifestList     = "application/vnd.docker.distribution.manifest.list.v2+json"
 	// Checkpoint/Restore Media Types
 	MediaTypeContainerd1Checkpoint        = "application/vnd.containerd.container.criu.checkpoint.criu.tar"
 	MediaTypeContainerd1CheckpointPreDump = "application/vnd.containerd.container.criu.checkpoint.predump.tar"

remotes/docker/fetcher.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/log"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -26,7 +27,7 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R
 		},
 	))
 
-	paths, err := getV2URLPaths(desc)
+	urls, err := r.getV2URLPaths(ctx, desc)
 	if err != nil {
 		return nil, err
 	}
@@ -36,9 +37,7 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R
 		return nil, err
 	}
 
-	for _, path := range paths {
+	for _, u := range urls {
-		u := r.url(path)
-
 		req, err := http.NewRequest(http.MethodGet, u, nil)
 		if err != nil {
 			return nil, err
@@ -61,24 +60,34 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R
 		return resp.Body, nil
 	}
 
-	return nil, errors.New("not found")
+	return nil, errors.Wrapf(errdefs.ErrNotFound,
+		"could not fetch content descriptor %v (%v) from remote",
+		desc.Digest, desc.MediaType)
 }
 
 // getV2URLPaths generates the candidate urls paths for the object based on the
 // set of hints and the provided object id. URLs are returned in the order of
 // most to least likely succeed.
-func getV2URLPaths(desc ocispec.Descriptor) ([]string, error) {
+func (r *dockerFetcher) getV2URLPaths(ctx context.Context, desc ocispec.Descriptor) ([]string, error) {
 	var urls []string
 
+	if len(desc.URLs) > 0 {
+		// handle fetch via external urls.
+		for _, u := range desc.URLs {
+			log.G(ctx).WithField("url", u).Debug("adding alternative url")
+			urls = append(urls, u)
+		}
+	}
+
 	switch desc.MediaType {
 	case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList,
 		images.MediaTypeDockerSchema1Manifest,
 		ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex:
-		urls = append(urls, path.Join("manifests", desc.Digest.String()))
+		urls = append(urls, r.url(path.Join("manifests", desc.Digest.String())))
 	}
 
 	// always fallback to attempting to get the object out of the blobs store.
-	urls = append(urls, path.Join("blobs", desc.Digest.String()))
+	urls = append(urls, r.url(path.Join("blobs", desc.Digest.String())))
 
 	return urls, nil
 }

remotes/handlers.go

@@ -27,6 +27,7 @@ func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string {
 	case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
 		return "index-" + desc.Digest.String()
 	case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip,
+		images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip,
 		ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
 		ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip:
 		return "layer-" + desc.Digest.String()