github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Generalize media types

Browse Source 
Avoid directly handling media types with "+" attributes,
instead handling the base and passing through the full
media type to the appropriate stream processor or decompression.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
This commit is contained in:
Derek McGowan 2019-09-12 15:57:14 -07:00
parent d4802a64f9
commit 6f31417d49
No known key found for this signature in database
GPG Key ID: F58C5D0A4405ACDB
5 changed files with 101 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
diff/stream.go
View File

@ -88,11 +88,11 @@ type StreamProcessor interface {
}
func compressedHandler(ctx context.Context, mediaType string) (StreamProcessorInit, bool) {
compressed, err := images.IsCompressedDiff(ctx, mediaType)
compressed, err := images.DiffCompression(ctx, mediaType)
if err != nil {
return nil, false
}
if compressed {
if compressed != "" {
return func(ctx context.Context, stream StreamProcessor, payloads map[string]*types.Any) (StreamProcessor, error) {
ds, err := compression.DecompressStream(stream)
if err != nil {

83
images/image.go
View File

@ -20,7 +20,6 @@ import (
"context"
"encoding/json"
"sort"
"strings"
"time"
"github.com/containerd/containerd/content"
@ -358,16 +357,11 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr
}
descs = append(descs, index.Manifests...)
case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
MediaTypeDockerSchema2LayerEnc, MediaTypeDockerSchema2LayerGzipEnc,
MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip,
MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig:
// childless data types.
return nil, nil
default:
if IsLayerType(desc.MediaType) || IsKnownConfig(desc.MediaType) {
// childless data types.
return nil, nil
}
log.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType)
}
@ -390,72 +384,3 @@ func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.D
}
return config.RootFS.DiffIDs, nil
}
// IsCompressedDiff returns true if mediaType is a known compressed diff media type.
// It returns false if the media type is a diff, but not compressed. If the media type
// is not a known diff type, it returns errdefs.ErrNotImplemented
func IsCompressedDiff(ctx context.Context, mediaType string) (bool, error) {
switch mediaType {
case ocispec.MediaTypeImageLayer, MediaTypeDockerSchema2Layer:
case ocispec.MediaTypeImageLayerGzip, MediaTypeDockerSchema2LayerGzip:
return true, nil
default:
// Still apply all generic media types *.tar[.+]gzip and *.tar
if strings.HasSuffix(mediaType, ".tar.gzip") || strings.HasSuffix(mediaType, ".tar+gzip") {
return true, nil
} else if !strings.HasSuffix(mediaType, ".tar") {
return false, errdefs.ErrNotImplemented
}
}
return false, nil
}
// GetImageLayerDescriptors gets the image layer Descriptors of an image; the array contains
// a list of Descriptors belonging to one platform followed by lists of other platforms
func GetImageLayerDescriptors(ctx context.Context, cs content.Store, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
var lis []ocispec.Descriptor
ds := platforms.DefaultSpec()
platform := &ds
switch desc.MediaType {
case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex,
MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
children, err := Children(ctx, cs, desc)
if err != nil {
if errdefs.IsNotFound(err) {
return []ocispec.Descriptor{}, nil
}
return []ocispec.Descriptor{}, err
}
if desc.Platform != nil {
platform = desc.Platform
}
for _, child := range children {
var tmp []ocispec.Descriptor
switch child.MediaType {
case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2Layer,
ocispec.MediaTypeImageLayerGzip, ocispec.MediaTypeImageLayer,
MediaTypeDockerSchema2LayerGzipEnc, MediaTypeDockerSchema2LayerEnc:
tdesc := child
tdesc.Platform = platform
tmp = append(tmp, tdesc)
default:
tmp, err = GetImageLayerDescriptors(ctx, cs, child)
}
if err != nil {
return []ocispec.Descriptor{}, err
}
lis = append(lis, tmp...)
}
case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
default:
return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "GetImageLayerInfo: unhandled media type %s", desc.MediaType)
}
return lis, nil
}

86
images/mediatypes.go
View File

@ -16,16 +16,23 @@
package images
import (
"context"
"sort"
"strings"
"github.com/containerd/containerd/errdefs"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// mediatype definitions for image components handled in containerd.
//
// oci components are generally referenced directly, although we may centralize
// here for clarity.
const (
MediaTypeDockerSchema2Layer = "application/vnd.docker.image.rootfs.diff.tar"
MediaTypeDockerSchema2LayerEnc = "application/vnd.docker.image.rootfs.diff.tar+enc"
MediaTypeDockerSchema2LayerForeign = "application/vnd.docker.image.rootfs.foreign.diff.tar"
MediaTypeDockerSchema2LayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip"
MediaTypeDockerSchema2LayerGzipEnc = "application/vnd.docker.image.rootfs.diff.tar.gzip+enc"
MediaTypeDockerSchema2LayerForeignGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
MediaTypeDockerSchema2Config = "application/vnd.docker.container.image.v1+json"
MediaTypeDockerSchema2Manifest = "application/vnd.docker.distribution.manifest.v2+json"
@ -42,3 +49,78 @@ const (
// Legacy Docker schema1 manifest
MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws"
)
// DiffCompression returns the compression as defined by the layer diff media
// type. For Docker media types without compression, "unknown" is returned to
// indicate that the media type may be compressed. If the media type is not
// recognized as a layer diff, then it returns errdefs.ErrNotImplemented
func DiffCompression(ctx context.Context, mediaType string) (string, error) {
base, ext := parseMediaTypes(mediaType)
switch base {
case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerForeign:
if len(ext) > 0 {
// Type is wrapped
return "", nil
}
// These media types may have been compressed but failed to
// use the correct media type. The decompression function
// should detect and handle this case.
return "unknown", nil
case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2LayerForeignGzip:
if len(ext) > 0 {
// Type is wrapped
return "", nil
}
return "gzip", nil
case ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerNonDistributable:
if len(ext) > 0 {
switch ext[len(ext)-1] {
case "gzip":
return "gzip", nil
}
}
return "", nil
default:
return "", errdefs.ErrNotImplemented
}
}
// parseMediaTypes splits the media type into the base type and
// an array of sorted extensions
func parseMediaTypes(mt string) (string, []string) {
if mt == "" {
return "", []string{}
}
s := strings.Split(mt, "+")
ext := s[1:]
sort.Strings(ext)
return s[0], ext
}
// IsLayerTypes returns true if the media type is a layer
func IsLayerType(mt string) bool {
if strings.HasPrefix(mt, "application/vnd.oci.image.layer.") {
return true
}
// Parse Docker media types, strip off any + suffixes first
base, _ := parseMediaTypes(mt)
switch base {
case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip:
return true
}
return false
}
// IsKnownConfig returns true if the media type is a known config type
func IsKnownConfig(mt string) bool {
switch mt {
case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig:
return true
}
return false
}

16
remotes/handlers.go
View File

@ -62,21 +62,17 @@ func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string {
}
}
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
switch mt := desc.MediaType; {
case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest:
return "manifest-" + desc.Digest.String()
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
case mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex:
return "index-" + desc.Digest.String()
case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip,
images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip,
ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc:
case images.IsLayerType(mt):
return "layer-" + desc.Digest.String()
case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
case images.IsKnownConfig(mt):
return "config-" + desc.Digest.String()
default:
log.G(ctx).Warnf("reference for unknown type: %s", desc.MediaType)
log.G(ctx).Warnf("reference for unknown type: %s", mt)
return "unknown-" + desc.Digest.String()
}
}

14
unpacker.go
View File

@ -204,12 +204,12 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im
// one manifest to handle, and manifest list can be
// safely skipped.
// TODO: support multi-platform unpack.
switch desc.MediaType {
case images.MediaTypeDockerSchema1Manifest:
switch mt := desc.MediaType; {
case mt == images.MediaTypeDockerSchema1Manifest:
lock.Lock()
schema1 = true
lock.Unlock()
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest:
lock.Lock()
for _, child := range children {
if child.MediaType == images.MediaTypeDockerSchema2Config ||
@ -219,7 +219,7 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im
layers = append(layers, child)
}
lock.Unlock()
case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
case mt == images.MediaTypeDockerSchema2Config || mt == ocispec.MediaTypeImageConfig:
lock.Lock()
l := append([]ocispec.Descriptor{}, layers...)
lock.Unlock()
@ -229,11 +229,7 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im
return u.unpack(uctx, desc, l)
})
}
case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip,
images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip,
ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc:
case images.IsLayerType(mt):
lock.Lock()
update := !schema1
lock.Unlock()