From bce20b75da6264f5f26e26130e9a0d52c006ecfd Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 11 Sep 2018 18:34:25 -0700 Subject: [PATCH] Simplify docker importer Support any layout and rely on manifest.json to reference blobs Signed-off-by: Derek McGowan --- cmd/ctr/commands/images/import.go | 11 +- images/docker/importer.go | 351 +++++++++--------------------- images/docker/reference.go | 65 ++++++ images/oci/importer.go | 9 - import_test.go | 3 +- 5 files changed, 179 insertions(+), 260 deletions(-) create mode 100644 images/docker/reference.go diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index 8b6bfcc2b..eab50d215 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "os" + "strings" "time" "github.com/containerd/containerd" @@ -81,15 +82,19 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb prefix := context.String("base-name") if prefix == "" { prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) + opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, false))) + } else { + // When provided, filter out references which do not match + opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, true))) } switch format := context.String("format"); format { case "", "docker", "docker.v1.1", "docker.v1.2": - opts = append(opts, containerd.WithImporter(&docker.V1Importer{})) - opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, context.String("base-name") != ""))) + opts = append(opts, containerd.WithImporter(&docker.V1Importer{ + SkipOCI: strings.HasPrefix(format, "docker"), + })) case "oci", "oci.v1": opts = append(opts, containerd.WithImporter(&oci.V1Importer{})) - opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix))) default: return fmt.Errorf("unknown format %s", format) } diff --git a/images/docker/importer.go b/images/docker/importer.go index 8d46287f2..5ddfa81d4 100644 --- a/images/docker/importer.go +++ b/images/docker/importer.go @@ -26,21 +26,24 @@ import ( "io" "io/ioutil" "path" - "strings" + "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" - "github.com/containerd/cri/pkg/util" digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) -// V1Importer implements OCI Image Spec v1. +// V1Importer implements Docker v1.1, v1.2 and OCI v1. type V1Importer struct { + // SkipOCI prevent interpretting OCI files + SkipOCI bool + // TODO: Add option to compress layers on ingest + } var _ images.Importer = &V1Importer{} @@ -48,14 +51,16 @@ var _ images.Importer = &V1Importer{} // Import implements Importer. func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { var ( - desc ocispec.Descriptor - tr = tar.NewReader(reader) - - mfsts []manifestDotJSON - symlinkLayers = make(map[string]string) // key: filename (foobar/layer.tar), value: linkname (targetlayerid/layer.tar) - layers = make(map[string]ocispec.Descriptor) // key: filename (foobar/layer.tar) - configs = make(map[string]imageConfig) // key: filename (foobar.json) + tr = tar.NewReader(reader) + ociLayout ocispec.ImageLayout + mfsts []struct { + Config string + RepoTags []string + Layers []string + } + symlinks = make(map[string]string) + blobs = make(map[string]ocispec.Descriptor) ) for { hdr, err := tr.Next() @@ -65,77 +70,70 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io if err != nil { return ocispec.Descriptor{}, err } - if hdr.Typeflag == tar.TypeSymlink && isLayerTar(hdr.Name) { - linkname, err := followSymlinkLayer(hdr.Linkname) - if err != nil { - log.G(ctx).WithError(err).WithField("file", hdr.Name).Debugf("symlink to %s ignored", hdr.Linkname) - } else { - symlinkLayers[hdr.Name] = linkname - } - continue + if hdr.Typeflag == tar.TypeSymlink { + symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname) } if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { - log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") + if hdr.Typeflag != tar.TypeDir { + log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") + } continue } + hdrName := path.Clean(hdr.Name) - if hdrName == "index.json" { - if desc.Digest != "" { - return ocispec.Descriptor{}, errors.New("duplicated index.json") + if hdrName == ocispec.ImageLayoutFile && !oi.SkipOCI { + if err = onUntarJSON(tr, &ociLayout); err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name) } - desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size) - if err != nil { - return ocispec.Descriptor{}, err - } - } else if strings.HasPrefix(hdrName, "blobs/") { - if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { - return ocispec.Descriptor{}, err - } - } else if hdrName == ocispec.ImageLayoutFile { - // TODO Validate - } else if hdr.Name == "manifest.json" { - mfsts, err = onUntarManifestJSON(tr) - if err != nil { + } else if hdrName == "manifest.json" { + if err = onUntarJSON(tr, &mfsts); err != nil { return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name) } - } else if isLayerTar(hdr.Name) { - desc, err := onUntarLayerTar(ctx, tr, store, hdr.Name, hdr.Size) - if err != nil { - return ocispec.Descriptor{}, errors.Wrapf(err, "untar layer %q", hdr.Name) - } - layers[hdr.Name] = desc - } else if isDotJSON(hdr.Name) { - c, err := onUntarDotJSON(ctx, tr, store, hdr.Name, hdr.Size) - if err != nil { - return ocispec.Descriptor{}, errors.Wrapf(err, "untar config %q", hdr.Name) - } - configs[hdr.Name] = c } else { - log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored") + dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "failed to ingest %q", hdr.Name) + } + + blobs[hdrName] = ocispec.Descriptor{ + Digest: dgst, + Size: hdr.Size, + } + } + } + + if ociLayout.Version != "" { + if ociLayout.Version != ocispec.ImageLayoutVersion { + return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version) } - } - if desc.Digest != "" { - return desc, nil + idx, ok := blobs["index.json"] + if !ok { + return ocispec.Descriptor{}, errors.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion) + } + + idx.MediaType = ocispec.MediaTypeImageIndex + return idx, nil } - for name, linkname := range symlinkLayers { - desc, ok := layers[linkname] + for name, linkname := range symlinks { + desc, ok := blobs[linkname] if !ok { return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname) } - layers[name] = desc + blobs[name] = desc } var idx ocispec.Index for _, mfst := range mfsts { - config, ok := configs[mfst.Config] + config, ok := blobs[mfst.Config] if !ok { return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config) } + config.MediaType = ocispec.MediaTypeImageConfig - layers, err := resolveLayers(mfst.Layers, layers) + layers, err := resolveLayers(ctx, store, mfst.Layers, blobs) if err != nil { return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers") } @@ -144,15 +142,24 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io Versioned: specs.Versioned{ SchemaVersion: 2, }, - Config: config.desc, + Config: config, Layers: layers, } - desc, err := writeManifest(ctx, store, manifest, config.img.Architecture, config.img.OS) + desc, err := writeManifest(ctx, store, manifest, ocispec.MediaTypeImageManifest) if err != nil { return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest") } + platforms, err := images.Platforms(ctx, store, desc) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform") + } + if len(platforms) > 0 { + // Only one platform can be resolved from non-index manifest + desc.Platform = &platforms[0] + } + if len(mfst.RepoTags) == 0 { idx.Manifests = append(idx.Manifests, desc) } else { @@ -160,14 +167,13 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io for _, ref := range mfst.RepoTags { msftdesc := desc - // TODO: Replace this function to not depend on reference package - normalized, err := util.NormalizeImageRef(ref) + normalized, err := normalizeReference(ref) if err != nil { - return ocispec.Descriptor{}, errors.Wrapf(err, "normalize image ref %q", ref) + return ocispec.Descriptor{}, err } msftdesc.Annotations = map[string]string{ - ocispec.AnnotationRefName: normalized.String(), + ocispec.AnnotationRefName: normalized, } idx.Manifests = append(idx.Manifests, msftdesc) @@ -175,117 +181,68 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io } } - return writeIndex(ctx, store, idx) + return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex) } -// RefTranslator creates a reference which only has a tag or verifies -// a full reference. -func RefTranslator(image string, checkPrefix bool) func(string) string { - return func(ref string) string { - // Check if ref is full reference - if strings.ContainsAny(ref, "/:@") { - // If not prefixed, don't include image - if checkPrefix && !isImagePrefix(ref, image) { - return "" - } - return ref - } - return image + ":" + ref - } -} - -func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) { +func onUntarJSON(r io.Reader, j interface{}) error { b, err := ioutil.ReadAll(r) if err != nil { - return ocispec.Descriptor{}, err + return err } - desc := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageIndex, - Digest: digest.FromBytes(b), - Size: size, + if err := json.Unmarshal(b, j); err != nil { + return err } - if int64(len(b)) != size { - return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size) - } - - if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil { - return ocispec.Descriptor{}, err - } - - return desc, err + return nil } -func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error { - // name is like "blobs/sha256/deadbeef" - split := strings.Split(name, "/") - if len(split) != 3 { - return errors.Errorf("unexpected name: %q", name) +func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) { + dgstr := digest.Canonical.Digester() + + if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil { + return "", err } - algo := digest.Algorithm(split[1]) - if !algo.Available() { - return errors.Errorf("unsupported algorithm: %s", algo) - } - dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) + + return dgstr.Digest(), nil } -// manifestDotJSON is an entry in manifest.json. -type manifestDotJSON struct { - Config string - RepoTags []string - Layers []string - // Parent is unsupported - Parent string -} - -// isLayerTar returns true if name is like "foobar/layer.tar" -func isLayerTar(name string) bool { - slashes := len(strings.Split(name, "/")) - return slashes == 2 && strings.HasSuffix(name, "/layer.tar") -} - -// followSymlinkLayer returns actual layer name of the symlink layer. -// It returns "foobar/layer.tar" if the name is like -// "../foobar/layer.tar", and returns error if the name -// is not in "../foobar/layer.tar" format. -func followSymlinkLayer(name string) (string, error) { - parts := strings.Split(name, "/") - if len(parts) != 3 || parts[0] != ".." { - return "", errors.New("invalid symlink layer") - } - name = strings.TrimPrefix(name, "../") - if !isLayerTar(name) { - return "", errors.New("invalid layer tar") - } - return name, nil -} - -// isDotJSON returns true if name is like "foobar.json" -func isDotJSON(name string) bool { - slashes := len(strings.Split(name, "/")) - return slashes == 1 && strings.HasSuffix(name, ".json") -} - -func resolveLayers(layerIDs []string, layerIDMap map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) { +func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) { var layers []ocispec.Descriptor - for _, f := range layerIDs { - desc, ok := layerIDMap[f] + for _, f := range layerFiles { + desc, ok := blobs[f] if !ok { return nil, errors.Errorf("layer %q not found", f) } + + // Open blob, resolve media type + ra, err := store.ReaderAt(ctx, desc) + if err != nil { + return nil, errors.Wrapf(err, "failed to open %q (%s)", f, desc.Digest) + } + s, err := compression.DecompressStream(content.NewReader(ra)) + if err != nil { + return nil, errors.Wrapf(err, "failed to detect compression for %q", f) + } + if s.GetCompression() == compression.Uncompressed { + // TODO: Support compressing and writing back to content store + desc.MediaType = ocispec.MediaTypeImageLayer + } else { + desc.MediaType = ocispec.MediaTypeImageLayerGzip + } + s.Close() + layers = append(layers, desc) } return layers, nil } -func writeManifest(ctx context.Context, cs content.Ingester, manifest ocispec.Manifest, arch, os string) (ocispec.Descriptor, error) { +func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) { manifestBytes, err := json.Marshal(manifest) if err != nil { return ocispec.Descriptor{}, err } desc := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageManifest, + MediaType: mediaType, Digest: digest.FromBytes(manifestBytes), Size: int64(len(manifestBytes)), } @@ -293,105 +250,5 @@ func writeManifest(ctx context.Context, cs content.Ingester, manifest ocispec.Ma return ocispec.Descriptor{}, err } - if arch != "" || os != "" { - desc.Platform = &ocispec.Platform{ - Architecture: arch, - OS: os, - } - } return desc, nil } - -func writeIndex(ctx context.Context, cs content.Ingester, manifest ocispec.Index) (ocispec.Descriptor, error) { - manifestBytes, err := json.Marshal(manifest) - if err != nil { - return ocispec.Descriptor{}, err - } - - desc := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageIndex, - Digest: digest.FromBytes(manifestBytes), - Size: int64(len(manifestBytes)), - } - if err := content.WriteBlob(ctx, cs, "index-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil { - return ocispec.Descriptor{}, err - } - - return desc, nil -} - -func onUntarManifestJSON(r io.Reader) ([]manifestDotJSON, error) { - // name: "manifest.json" - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - var mfsts []manifestDotJSON - if err := json.Unmarshal(b, &mfsts); err != nil { - return nil, err - } - return mfsts, nil -} - -func onUntarLayerTar(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (ocispec.Descriptor, error) { - // name is like "foobar/layer.tar" ( guaranteed by isLayerTar() ) - split := strings.Split(name, "/") - // note: split[0] is not expected digest here - cw, err := cs.Writer(ctx, content.WithRef("layer-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size})) - if err != nil { - return ocispec.Descriptor{}, err - } - // TODO: support compression and committing with labels - defer cw.Close() - if err := content.Copy(ctx, cw, r, size, ""); err != nil { - return ocispec.Descriptor{}, err - } - return ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageLayer, - Size: size, - Digest: cw.Digest(), - }, nil -} - -type imageConfig struct { - desc ocispec.Descriptor - img ocispec.Image -} - -func onUntarDotJSON(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (imageConfig, error) { - config := imageConfig{} - config.desc.MediaType = ocispec.MediaTypeImageConfig - config.desc.Size = size - // name is like "foobar.json" ( guaranteed by is DotJSON() ) - split := strings.Split(name, ".") - cw, err := cs.Writer(ctx, content.WithRef("config-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size})) - if err != nil { - return imageConfig{}, err - } - defer cw.Close() - var buf bytes.Buffer - tr := io.TeeReader(r, &buf) - if err := content.Copy(ctx, cw, tr, size, ""); err != nil { - return imageConfig{}, err - } - config.desc.Digest = cw.Digest() - if err := json.Unmarshal(buf.Bytes(), &config.img); err != nil { - return imageConfig{}, err - } - return config, nil -} - -func isImagePrefix(s, prefix string) bool { - if !strings.HasPrefix(s, prefix) { - return false - } - if len(s) > len(prefix) { - switch s[len(prefix)] { - case '/', ':', '@': - // Prevent matching partial namespaces - default: - return false - } - } - return true -} diff --git a/images/docker/reference.go b/images/docker/reference.go new file mode 100644 index 000000000..ed7d7d91f --- /dev/null +++ b/images/docker/reference.go @@ -0,0 +1,65 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package docker + +import ( + "strings" + + "github.com/containerd/cri/pkg/util" + "github.com/pkg/errors" +) + +// RefTranslator creates a reference which only has a tag or verifies +// a full reference. +func RefTranslator(image string, checkPrefix bool) func(string) string { + return func(ref string) string { + // Check if ref is full reference + if strings.ContainsAny(ref, "/:@") { + // If not prefixed, don't include image + if checkPrefix && !isImagePrefix(ref, image) { + return "" + } + return ref + } + return image + ":" + ref + } +} + +func isImagePrefix(s, prefix string) bool { + if !strings.HasPrefix(s, prefix) { + return false + } + if len(s) > len(prefix) { + switch s[len(prefix)] { + case '/', ':', '@': + // Prevent matching partial namespaces + default: + return false + } + } + return true +} + +func normalizeReference(ref string) (string, error) { + // TODO: Replace this function to not depend on reference package + normalized, err := util.NormalizeImageRef(ref) + if err != nil { + return "", errors.Wrapf(err, "normalize image ref %q", ref) + } + + return normalized.String(), nil +} diff --git a/images/oci/importer.go b/images/oci/importer.go index 391d5b9f3..6c2b9803f 100644 --- a/images/oci/importer.go +++ b/images/oci/importer.go @@ -118,15 +118,6 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) } -// RefTranslator creates a reference using an OCI ref annotation, -// which is mentioned in the spec as only a tag compontent, -// concatenated with an image name -func RefTranslator(prefix string) func(string) string { - return func(ref string) string { - return prefix + ":" + ref - } -} - // DigestTranslator creates a digest reference by adding the // digest to an image name func DigestTranslator(prefix string) func(digest.Digest) string { diff --git a/import_test.go b/import_test.go index 10205baca..d72d47eb2 100644 --- a/import_test.go +++ b/import_test.go @@ -20,6 +20,7 @@ import ( "runtime" "testing" + "github.com/containerd/containerd/images/docker" "github.com/containerd/containerd/images/oci" ) @@ -51,7 +52,7 @@ func TestOCIExportAndImport(t *testing.T) { opts := []ImportOpt{ WithImporter(&oci.V1Importer{}), - WithImageRefTranslator(oci.RefTranslator("foo/bar")), + WithImageRefTranslator(docker.RefTranslator("foo/bar", false)), } imgrecs, err := client.Import(ctx, exported, opts...) if err != nil {