From 6f31417d4980d34fd8042c0b91a904b5feeb5337 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 12 Sep 2019 15:57:14 -0700 Subject: [PATCH] Generalize media types Avoid directly handling media types with "+" attributes, instead handling the base and passing through the full media type to the appropriate stream processor or decompression. Signed-off-by: Derek McGowan --- diff/stream.go | 4 +-- images/image.go | 83 +++--------------------------------------- images/mediatypes.go | 86 ++++++++++++++++++++++++++++++++++++++++++-- remotes/handlers.go | 16 ++++----- unpacker.go | 14 +++----- 5 files changed, 101 insertions(+), 102 deletions(-) diff --git a/diff/stream.go b/diff/stream.go index 4b8f27f14..1b625feaa 100644 --- a/diff/stream.go +++ b/diff/stream.go @@ -88,11 +88,11 @@ type StreamProcessor interface { } func compressedHandler(ctx context.Context, mediaType string) (StreamProcessorInit, bool) { - compressed, err := images.IsCompressedDiff(ctx, mediaType) + compressed, err := images.DiffCompression(ctx, mediaType) if err != nil { return nil, false } - if compressed { + if compressed != "" { return func(ctx context.Context, stream StreamProcessor, payloads map[string]*types.Any) (StreamProcessor, error) { ds, err := compression.DecompressStream(stream) if err != nil { diff --git a/images/image.go b/images/image.go index 7d4f39c0a..ee5778d24 100644 --- a/images/image.go +++ b/images/image.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "sort" - "strings" "time" "github.com/containerd/containerd/content" @@ -358,16 +357,11 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr } descs = append(descs, index.Manifests...) - case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip, - MediaTypeDockerSchema2LayerEnc, MediaTypeDockerSchema2LayerGzipEnc, - MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip, - MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig, - ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip, - ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip, - MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig: - // childless data types. - return nil, nil default: + if IsLayerType(desc.MediaType) || IsKnownConfig(desc.MediaType) { + // childless data types. + return nil, nil + } log.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType) } @@ -390,72 +384,3 @@ func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.D } return config.RootFS.DiffIDs, nil } - -// IsCompressedDiff returns true if mediaType is a known compressed diff media type. -// It returns false if the media type is a diff, but not compressed. If the media type -// is not a known diff type, it returns errdefs.ErrNotImplemented -func IsCompressedDiff(ctx context.Context, mediaType string) (bool, error) { - switch mediaType { - case ocispec.MediaTypeImageLayer, MediaTypeDockerSchema2Layer: - case ocispec.MediaTypeImageLayerGzip, MediaTypeDockerSchema2LayerGzip: - return true, nil - default: - // Still apply all generic media types *.tar[.+]gzip and *.tar - if strings.HasSuffix(mediaType, ".tar.gzip") || strings.HasSuffix(mediaType, ".tar+gzip") { - return true, nil - } else if !strings.HasSuffix(mediaType, ".tar") { - return false, errdefs.ErrNotImplemented - } - } - return false, nil -} - -// GetImageLayerDescriptors gets the image layer Descriptors of an image; the array contains -// a list of Descriptors belonging to one platform followed by lists of other platforms -func GetImageLayerDescriptors(ctx context.Context, cs content.Store, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - var lis []ocispec.Descriptor - - ds := platforms.DefaultSpec() - platform := &ds - - switch desc.MediaType { - case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex, - MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - children, err := Children(ctx, cs, desc) - if err != nil { - if errdefs.IsNotFound(err) { - return []ocispec.Descriptor{}, nil - } - return []ocispec.Descriptor{}, err - } - - if desc.Platform != nil { - platform = desc.Platform - } - - for _, child := range children { - var tmp []ocispec.Descriptor - - switch child.MediaType { - case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2Layer, - ocispec.MediaTypeImageLayerGzip, ocispec.MediaTypeImageLayer, - MediaTypeDockerSchema2LayerGzipEnc, MediaTypeDockerSchema2LayerEnc: - tdesc := child - tdesc.Platform = platform - tmp = append(tmp, tdesc) - default: - tmp, err = GetImageLayerDescriptors(ctx, cs, child) - } - - if err != nil { - return []ocispec.Descriptor{}, err - } - - lis = append(lis, tmp...) - } - case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: - default: - return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "GetImageLayerInfo: unhandled media type %s", desc.MediaType) - } - return lis, nil -} diff --git a/images/mediatypes.go b/images/mediatypes.go index 5fee7467a..2f47b0e68 100644 --- a/images/mediatypes.go +++ b/images/mediatypes.go @@ -16,16 +16,23 @@ package images +import ( + "context" + "sort" + "strings" + + "github.com/containerd/containerd/errdefs" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + // mediatype definitions for image components handled in containerd. // // oci components are generally referenced directly, although we may centralize // here for clarity. const ( MediaTypeDockerSchema2Layer = "application/vnd.docker.image.rootfs.diff.tar" - MediaTypeDockerSchema2LayerEnc = "application/vnd.docker.image.rootfs.diff.tar+enc" MediaTypeDockerSchema2LayerForeign = "application/vnd.docker.image.rootfs.foreign.diff.tar" MediaTypeDockerSchema2LayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip" - MediaTypeDockerSchema2LayerGzipEnc = "application/vnd.docker.image.rootfs.diff.tar.gzip+enc" MediaTypeDockerSchema2LayerForeignGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" MediaTypeDockerSchema2Config = "application/vnd.docker.container.image.v1+json" MediaTypeDockerSchema2Manifest = "application/vnd.docker.distribution.manifest.v2+json" @@ -42,3 +49,78 @@ const ( // Legacy Docker schema1 manifest MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" ) + +// DiffCompression returns the compression as defined by the layer diff media +// type. For Docker media types without compression, "unknown" is returned to +// indicate that the media type may be compressed. If the media type is not +// recognized as a layer diff, then it returns errdefs.ErrNotImplemented +func DiffCompression(ctx context.Context, mediaType string) (string, error) { + base, ext := parseMediaTypes(mediaType) + switch base { + case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerForeign: + if len(ext) > 0 { + // Type is wrapped + return "", nil + } + // These media types may have been compressed but failed to + // use the correct media type. The decompression function + // should detect and handle this case. + return "unknown", nil + case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2LayerForeignGzip: + if len(ext) > 0 { + // Type is wrapped + return "", nil + } + return "gzip", nil + case ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerNonDistributable: + if len(ext) > 0 { + switch ext[len(ext)-1] { + case "gzip": + return "gzip", nil + } + } + return "", nil + default: + return "", errdefs.ErrNotImplemented + } +} + +// parseMediaTypes splits the media type into the base type and +// an array of sorted extensions +func parseMediaTypes(mt string) (string, []string) { + if mt == "" { + return "", []string{} + } + + s := strings.Split(mt, "+") + ext := s[1:] + sort.Strings(ext) + + return s[0], ext +} + +// IsLayerTypes returns true if the media type is a layer +func IsLayerType(mt string) bool { + if strings.HasPrefix(mt, "application/vnd.oci.image.layer.") { + return true + } + + // Parse Docker media types, strip off any + suffixes first + base, _ := parseMediaTypes(mt) + switch base { + case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip, + MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip: + return true + } + return false +} + +// IsKnownConfig returns true if the media type is a known config type +func IsKnownConfig(mt string) bool { + switch mt { + case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig, + MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig: + return true + } + return false +} diff --git a/remotes/handlers.go b/remotes/handlers.go index 0a243eda1..671fea106 100644 --- a/remotes/handlers.go +++ b/remotes/handlers.go @@ -62,21 +62,17 @@ func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string { } } - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + switch mt := desc.MediaType; { + case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest: return "manifest-" + desc.Digest.String() - case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + case mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex: return "index-" + desc.Digest.String() - case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip, - images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip, - ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip, - ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip, - images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc: + case images.IsLayerType(mt): return "layer-" + desc.Digest.String() - case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: + case images.IsKnownConfig(mt): return "config-" + desc.Digest.String() default: - log.G(ctx).Warnf("reference for unknown type: %s", desc.MediaType) + log.G(ctx).Warnf("reference for unknown type: %s", mt) return "unknown-" + desc.Digest.String() } } diff --git a/unpacker.go b/unpacker.go index 790c06c8d..f7580ce85 100644 --- a/unpacker.go +++ b/unpacker.go @@ -204,12 +204,12 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im // one manifest to handle, and manifest list can be // safely skipped. // TODO: support multi-platform unpack. - switch desc.MediaType { - case images.MediaTypeDockerSchema1Manifest: + switch mt := desc.MediaType; { + case mt == images.MediaTypeDockerSchema1Manifest: lock.Lock() schema1 = true lock.Unlock() - case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest: lock.Lock() for _, child := range children { if child.MediaType == images.MediaTypeDockerSchema2Config || @@ -219,7 +219,7 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im layers = append(layers, child) } lock.Unlock() - case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: + case mt == images.MediaTypeDockerSchema2Config || mt == ocispec.MediaTypeImageConfig: lock.Lock() l := append([]ocispec.Descriptor{}, layers...) lock.Unlock() @@ -229,11 +229,7 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im return u.unpack(uctx, desc, l) }) } - case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip, - images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip, - ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip, - ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip, - images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc: + case images.IsLayerType(mt): lock.Lock() update := !schema1 lock.Unlock()