Generalize media types

Avoid directly handling media types with "+" attributes,
instead handling the base and passing through the full
media type to the appropriate stream processor or decompression.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

diff/stream.go

@@ -88,11 +88,11 @@ type StreamProcessor interface {
 }
 
 func compressedHandler(ctx context.Context, mediaType string) (StreamProcessorInit, bool) {
-	compressed, err := images.IsCompressedDiff(ctx, mediaType)
+	compressed, err := images.DiffCompression(ctx, mediaType)
 	if err != nil {
 		return nil, false
 	}
-	if compressed {
+	if compressed != "" {
 		return func(ctx context.Context, stream StreamProcessor, payloads map[string]*types.Any) (StreamProcessor, error) {
 			ds, err := compression.DecompressStream(stream)
 			if err != nil {

images/image.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"encoding/json"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/containerd/containerd/content"
@@ -358,16 +357,11 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr
 		}
 
 		descs = append(descs, index.Manifests...)
-	case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
-		MediaTypeDockerSchema2LayerEnc, MediaTypeDockerSchema2LayerGzipEnc,
-		MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip,
-		MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
-		ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
-		ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
-		MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig:
-		// childless data types.
-		return nil, nil
 	default:
+		if IsLayerType(desc.MediaType) || IsKnownConfig(desc.MediaType) {
+			// childless data types.
+			return nil, nil
+		}
 		log.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType)
 	}
 
@@ -390,72 +384,3 @@ func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.D
 	}
 	return config.RootFS.DiffIDs, nil
 }
-
-// IsCompressedDiff returns true if mediaType is a known compressed diff media type.
-// It returns false if the media type is a diff, but not compressed. If the media type
-// is not a known diff type, it returns errdefs.ErrNotImplemented
-func IsCompressedDiff(ctx context.Context, mediaType string) (bool, error) {
-	switch mediaType {
-	case ocispec.MediaTypeImageLayer, MediaTypeDockerSchema2Layer:
-	case ocispec.MediaTypeImageLayerGzip, MediaTypeDockerSchema2LayerGzip:
-		return true, nil
-	default:
-		// Still apply all generic media types *.tar[.+]gzip and *.tar
-		if strings.HasSuffix(mediaType, ".tar.gzip") || strings.HasSuffix(mediaType, ".tar+gzip") {
-			return true, nil
-		} else if !strings.HasSuffix(mediaType, ".tar") {
-			return false, errdefs.ErrNotImplemented
-		}
-	}
-	return false, nil
-}
-
-// GetImageLayerDescriptors gets the image layer Descriptors of an image; the array contains
-// a list of Descriptors belonging to one platform followed by lists of other platforms
-func GetImageLayerDescriptors(ctx context.Context, cs content.Store, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-	var lis []ocispec.Descriptor
-
-	ds := platforms.DefaultSpec()
-	platform := &ds
-
-	switch desc.MediaType {
-	case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex,
-		MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
-		children, err := Children(ctx, cs, desc)
-		if err != nil {
-			if errdefs.IsNotFound(err) {
-				return []ocispec.Descriptor{}, nil
-			}
-			return []ocispec.Descriptor{}, err
-		}
-
-		if desc.Platform != nil {
-			platform = desc.Platform
-		}
-
-		for _, child := range children {
-			var tmp []ocispec.Descriptor
-
-			switch child.MediaType {
-			case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2Layer,
-				ocispec.MediaTypeImageLayerGzip, ocispec.MediaTypeImageLayer,
-				MediaTypeDockerSchema2LayerGzipEnc, MediaTypeDockerSchema2LayerEnc:
-				tdesc := child
-				tdesc.Platform = platform
-				tmp = append(tmp, tdesc)
-			default:
-				tmp, err = GetImageLayerDescriptors(ctx, cs, child)
-			}
-
-			if err != nil {
-				return []ocispec.Descriptor{}, err
-			}
-
-			lis = append(lis, tmp...)
-		}
-	case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
-	default:
-		return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "GetImageLayerInfo: unhandled media type %s", desc.MediaType)
-	}
-	return lis, nil
-}

images/mediatypes.go

@@ -16,16 +16,23 @@
 
 package images
 
+import (
+	"context"
+	"sort"
+	"strings"
+
+	"github.com/containerd/containerd/errdefs"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
 // mediatype definitions for image components handled in containerd.
 //
 // oci components are generally referenced directly, although we may centralize
 // here for clarity.
 const (
 	MediaTypeDockerSchema2Layer            = "application/vnd.docker.image.rootfs.diff.tar"
-	MediaTypeDockerSchema2LayerEnc         = "application/vnd.docker.image.rootfs.diff.tar+enc"
 	MediaTypeDockerSchema2LayerForeign     = "application/vnd.docker.image.rootfs.foreign.diff.tar"
 	MediaTypeDockerSchema2LayerGzip        = "application/vnd.docker.image.rootfs.diff.tar.gzip"
-	MediaTypeDockerSchema2LayerGzipEnc     = "application/vnd.docker.image.rootfs.diff.tar.gzip+enc"
 	MediaTypeDockerSchema2LayerForeignGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
 	MediaTypeDockerSchema2Config           = "application/vnd.docker.container.image.v1+json"
 	MediaTypeDockerSchema2Manifest         = "application/vnd.docker.distribution.manifest.v2+json"
@@ -42,3 +49,78 @@ const (
 	// Legacy Docker schema1 manifest
 	MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws"
 )
+
+// DiffCompression returns the compression as defined by the layer diff media
+// type. For Docker media types without compression, "unknown" is returned to
+// indicate that the media type may be compressed. If the media type is not
+// recognized as a layer diff, then it returns errdefs.ErrNotImplemented
+func DiffCompression(ctx context.Context, mediaType string) (string, error) {
+	base, ext := parseMediaTypes(mediaType)
+	switch base {
+	case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerForeign:
+		if len(ext) > 0 {
+			// Type is wrapped
+			return "", nil
+		}
+		// These media types may have been compressed but failed to
+		// use the correct media type. The decompression function
+		// should detect and handle this case.
+		return "unknown", nil
+	case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2LayerForeignGzip:
+		if len(ext) > 0 {
+			// Type is wrapped
+			return "", nil
+		}
+		return "gzip", nil
+	case ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerNonDistributable:
+		if len(ext) > 0 {
+			switch ext[len(ext)-1] {
+			case "gzip":
+				return "gzip", nil
+			}
+		}
+		return "", nil
+	default:
+		return "", errdefs.ErrNotImplemented
+	}
+}
+
+// parseMediaTypes splits the media type into the base type and
+// an array of sorted extensions
+func parseMediaTypes(mt string) (string, []string) {
+	if mt == "" {
+		return "", []string{}
+	}
+
+	s := strings.Split(mt, "+")
+	ext := s[1:]
+	sort.Strings(ext)
+
+	return s[0], ext
+}
+
+// IsLayerTypes returns true if the media type is a layer
+func IsLayerType(mt string) bool {
+	if strings.HasPrefix(mt, "application/vnd.oci.image.layer.") {
+		return true
+	}
+
+	// Parse Docker media types, strip off any + suffixes first
+	base, _ := parseMediaTypes(mt)
+	switch base {
+	case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
+		MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip:
+		return true
+	}
+	return false
+}
+
+// IsKnownConfig returns true if the media type is a known config type
+func IsKnownConfig(mt string) bool {
+	switch mt {
+	case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
+		MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig:
+		return true
+	}
+	return false
+}

remotes/handlers.go

@@ -62,21 +62,17 @@ func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string {
 		}
 	}
 
-	switch desc.MediaType {
+	switch mt := desc.MediaType; {
-	case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+	case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest:
 		return "manifest-" + desc.Digest.String()
-	case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+	case mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex:
 		return "index-" + desc.Digest.String()
-	case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip,
+	case images.IsLayerType(mt):
-		images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip,
-		ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
-		ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
-		images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc:
 		return "layer-" + desc.Digest.String()
-	case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
+	case images.IsKnownConfig(mt):
 		return "config-" + desc.Digest.String()
 	default:
-		log.G(ctx).Warnf("reference for unknown type: %s", desc.MediaType)
+		log.G(ctx).Warnf("reference for unknown type: %s", mt)
 		return "unknown-" + desc.Digest.String()
 	}
 }

unpacker.go

@@ -204,12 +204,12 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im
 			// one manifest to handle, and manifest list can be
 			// safely skipped.
 			// TODO: support multi-platform unpack.
-			switch desc.MediaType {
+			switch mt := desc.MediaType; {
-			case images.MediaTypeDockerSchema1Manifest:
+			case mt == images.MediaTypeDockerSchema1Manifest:
 				lock.Lock()
 				schema1 = true
 				lock.Unlock()
-			case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+			case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest:
 				lock.Lock()
 				for _, child := range children {
 					if child.MediaType == images.MediaTypeDockerSchema2Config ||
@@ -219,7 +219,7 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im
 					layers = append(layers, child)
 				}
 				lock.Unlock()
-			case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
+			case mt == images.MediaTypeDockerSchema2Config || mt == ocispec.MediaTypeImageConfig:
 				lock.Lock()
 				l := append([]ocispec.Descriptor{}, layers...)
 				lock.Unlock()
@@ -229,11 +229,7 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im
 						return u.unpack(uctx, desc, l)
 					})
 				}
-			case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip,
+			case images.IsLayerType(mt):
-				images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip,
-				ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
-				ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
-				images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc:
 				lock.Lock()
 				update := !schema1
 				lock.Unlock()