From f57c5cdefbbcbd3b986be30ffdcda8375bc0dd28 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 6 Sep 2018 12:03:17 -0700 Subject: [PATCH 1/5] Refactor image importer Allow customization of reference creation. Add option for digest references. Signed-off-by: Derek McGowan --- cmd/ctr/commands/images/import.go | 44 +++++--- images/importexport.go | 2 +- images/oci/importer.go | 164 +++++++++--------------------- images/oci/importer_test.go | 53 ---------- import.go | 147 +++++++++++++++++++++----- import_test.go | 8 +- 6 files changed, 206 insertions(+), 212 deletions(-) delete mode 100644 images/oci/importer_test.go diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index 37c3ab5c7..b19131575 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -20,9 +20,10 @@ import ( "fmt" "io" "os" + "time" + "github.com/containerd/containerd" "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/images" oci "github.com/containerd/containerd/images/oci" "github.com/containerd/containerd/log" "github.com/urfave/cli" @@ -53,23 +54,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb Usage: "image format. See DESCRIPTION.", }, cli.StringFlag{ - Name: "oci-name", - Value: "unknown/unknown", - Usage: "prefix added to either oci.v1 ref annotation or digest", + Name: "prefix,oci-name", + Value: "", + Usage: "prefix image name for added images", + }, + cli.BoolFlag{ + Name: "digests", + Usage: "whether to create digest images", }, - // TODO(AkihiroSuda): support commands.LabelFlag (for all children objects) }, commands.SnapshotterFlags...), Action: func(context *cli.Context) error { var ( - in = context.Args().First() - imageImporter images.Importer + in = context.Args().First() + opts []containerd.ImportOpt ) switch format := context.String("format"); format { case "oci.v1": - imageImporter = &oci.V1Importer{ - ImageName: context.String("oci-name"), + opts = append(opts, containerd.WithImporter(&oci.V1Importer{})) + + prefix := context.String("prefix") + if prefix == "" { + prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) + } + + opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix))) + if context.Bool("digests") { + opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix))) } default: return fmt.Errorf("unknown format %s", format) @@ -90,20 +102,24 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb return err } } - imgs, err := client.Import(ctx, imageImporter, r) + imgs, err := client.Import(ctx, r, opts...) + closeErr := r.Close() if err != nil { return err } - if err = r.Close(); err != nil { - return err + if closeErr != nil { + return closeErr } log.G(ctx).Debugf("unpacking %d images", len(imgs)) for _, img := range imgs { + // TODO: Allow configuration of the platform + image := containerd.NewImage(client, img) + // TODO: Show unpack status - fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest) - err = img.Unpack(ctx, context.String("snapshotter")) + fmt.Printf("unpacking %s (%s)...", img.Name, img.Target.Digest) + err = image.Unpack(ctx, context.String("snapshotter")) if err != nil { return err } diff --git a/images/importexport.go b/images/importexport.go index 04a55fd38..843adcadc 100644 --- a/images/importexport.go +++ b/images/importexport.go @@ -27,7 +27,7 @@ import ( // Importer is the interface for image importer. type Importer interface { // Import imports an image from a tar stream. - Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error) + Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) } // Exporter is the interface for image exporter. diff --git a/images/oci/importer.go b/images/oci/importer.go index 64efd3fc2..391d5b9f3 100644 --- a/images/oci/importer.go +++ b/images/oci/importer.go @@ -19,114 +19,89 @@ package oci import ( "archive/tar" + "bytes" "context" - "encoding/json" - "fmt" "io" "io/ioutil" "path" "strings" "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/log" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) // V1Importer implements OCI Image Spec v1. -type V1Importer struct { - // ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs). - // This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355 - ImageName string -} +type V1Importer struct{} var _ images.Importer = &V1Importer{} // Import implements Importer. -func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) { - if oi.ImageName == "" { - return nil, errors.New("ImageName not set") - } - tr := tar.NewReader(reader) - var imgrecs []images.Image - foundIndexJSON := false +func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { + var ( + desc ocispec.Descriptor + tr = tar.NewReader(reader) + ) for { hdr, err := tr.Next() if err == io.EOF { break } if err != nil { - return nil, err + return ocispec.Descriptor{}, err } if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") continue } hdrName := path.Clean(hdr.Name) if hdrName == "index.json" { - if foundIndexJSON { - return nil, errors.New("duplicated index.json") + if desc.Digest != "" { + return ocispec.Descriptor{}, errors.New("duplicated index.json") } - foundIndexJSON = true - imgrecs, err = onUntarIndexJSON(tr, oi.ImageName) + desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size) if err != nil { - return nil, err + return ocispec.Descriptor{}, err } - continue - } - if strings.HasPrefix(hdrName, "blobs/") { + } else if strings.HasPrefix(hdrName, "blobs/") { if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { - return nil, err + return ocispec.Descriptor{}, err } + } else if hdrName == ocispec.ImageLayoutFile { + // TODO Validate + } else { + log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored") } } - if !foundIndexJSON { - return nil, errors.New("no index.json found") + if desc.Digest == "" { + return ocispec.Descriptor{}, errors.New("no index.json found") } - for _, img := range imgrecs { - err := setGCRefContentLabels(ctx, store, img.Target) - if err != nil { - return imgrecs, err - } - } - // FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)? - return imgrecs, nil + + return desc, nil } -func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) { +func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) { b, err := ioutil.ReadAll(r) if err != nil { - return nil, err + return ocispec.Descriptor{}, err } - var idx ocispec.Index - if err := json.Unmarshal(b, &idx); err != nil { - return nil, err + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageIndex, + Digest: digest.FromBytes(b), + Size: size, } - var imgrecs []images.Image - for _, m := range idx.Manifests { - ref, err := normalizeImageRef(imageName, m) - if err != nil { - return nil, err - } - imgrecs = append(imgrecs, images.Image{ - Name: ref, - Target: m, - }) + if int64(len(b)) != size { + return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size) } - return imgrecs, nil -} -func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) { - digest := manifest.Digest - if digest == "" { - return "", errors.Errorf("manifest with empty digest: %v", manifest) + if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil { + return ocispec.Descriptor{}, err } - ociRef := manifest.Annotations[ocispec.AnnotationRefName] - if ociRef == "" { - return imageName + "@" + digest.String(), nil - } - return imageName + ":" + ociRef, nil + + return desc, err } func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error { @@ -140,65 +115,22 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name return errors.Errorf("unsupported algorithm: %s", algo) } dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) + return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) } -// GetChildrenDescriptors returns children blob descriptors for the following supported types: -// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest -// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex -func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - var manifest ocispec.Manifest - if err := json.NewDecoder(r).Decode(&manifest); err != nil { - return nil, err - } - return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil - case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: - var index ocispec.Index - if err := json.NewDecoder(r).Decode(&index); err != nil { - return nil, err - } - return index.Manifests, nil +// RefTranslator creates a reference using an OCI ref annotation, +// which is mentioned in the spec as only a tag compontent, +// concatenated with an image name +func RefTranslator(prefix string) func(string) string { + return func(ref string) string { + return prefix + ":" + ref } - return nil, nil } -func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error { - info, err := store.Info(ctx, desc.Digest) - if err != nil { - if errdefs.IsNotFound(err) { - // when the archive is created from multi-arch image, - // it may contain only blobs for a certain platform. - // So ErrNotFound (on manifest list) is expected here. - return nil - } - return err +// DigestTranslator creates a digest reference by adding the +// digest to an image name +func DigestTranslator(prefix string) func(digest.Digest) string { + return func(dgst digest.Digest) string { + return prefix + "@" + dgst.String() } - ra, err := store.ReaderAt(ctx, desc) - if err != nil { - return err - } - defer ra.Close() - r := content.NewReader(ra) - children, err := GetChildrenDescriptors(r, desc) - if err != nil { - return err - } - if info.Labels == nil { - info.Labels = map[string]string{} - } - for i, child := range children { - // Note: child blob is not guaranteed to be written to the content store. (multi-arch) - info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String() - } - if _, err := store.Update(ctx, info, "labels"); err != nil { - return err - } - for _, child := range children { - if err := setGCRefContentLabels(ctx, store, child); err != nil { - return err - } - } - return nil } diff --git a/images/oci/importer_test.go b/images/oci/importer_test.go deleted file mode 100644 index e8c950597..000000000 --- a/images/oci/importer_test.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package oci - -import ( - "testing" - - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "gotest.tools/assert" -) - -func TestNormalizeImageRef(t *testing.T) { - imageBaseName := "foo/bar" - for _, test := range []struct { - input ocispec.Descriptor - expect string - }{ - { - input: ocispec.Descriptor{ - Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"), - }, - expect: "foo/bar@sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77", - }, - { - input: ocispec.Descriptor{ - Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"), - Annotations: map[string]string{ - ocispec.AnnotationRefName: "latest", - }, - }, - expect: "foo/bar:latest", // no @digest for simplicity - }, - } { - normalized, err := normalizeImageRef(imageBaseName, test.input) - assert.NilError(t, err) - assert.Equal(t, test.expect, normalized) - } -} diff --git a/import.go b/import.go index 7a69f1d45..4e42a7429 100644 --- a/import.go +++ b/import.go @@ -18,35 +18,75 @@ package containerd import ( "context" + "encoding/json" "io" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/oci" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) type importOpts struct { + indexName string + imageRefT func(string) string + dgstRefT func(digest.Digest) string + importer images.Importer } // ImportOpt allows the caller to specify import specific options -type ImportOpt func(c *importOpts) error +type ImportOpt func(*importOpts) error -func resolveImportOpt(opts ...ImportOpt) (importOpts, error) { - var iopts importOpts - for _, o := range opts { - if err := o(&iopts); err != nil { - return iopts, err - } +// WithImageRefTranslator is used to translate the index reference +// to an image reference for the image store. +func WithImageRefTranslator(f func(string) string) ImportOpt { + return func(c *importOpts) error { + c.imageRefT = f + return nil + } +} + +// WithDigestRef is used to create digest images for each +// manifest in the index. +func WithDigestRef(f func(digest.Digest) string) ImportOpt { + return func(c *importOpts) error { + c.dgstRefT = f + return nil + } +} + +// WithIndexName creates a tag pointing to the imported index +func WithIndexName(name string) ImportOpt { + return func(c *importOpts) error { + c.indexName = name + return nil + } +} + +// WithImporter sets the importer to use for converting +// the read stream into an OCI Index. +func WithImporter(importer images.Importer) ImportOpt { + return func(c *importOpts) error { + c.importer = importer + return nil } - return iopts, nil } // Import imports an image from a Tar stream using reader. // Caller needs to specify importer. Future version may use oci.v1 as the default. // Note that unreferrenced blobs may be imported to the content store as well. -func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) { - _, err := resolveImportOpt(opts...) // unused now - if err != nil { - return nil, err +func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt) ([]images.Image, error) { + var iopts importOpts + for _, o := range opts { + if err := o(&iopts); err != nil { + return nil, err + } + } + + if iopts.importer == nil { + iopts.importer = &oci.V1Importer{} } ctx, done, err := c.WithLease(ctx) @@ -55,31 +95,86 @@ func (c *Client) Import(ctx context.Context, importer images.Importer, reader io } defer done(ctx) - imgrecs, err := importer.Import(ctx, c.ContentStore(), reader) + index, err := iopts.importer.Import(ctx, c.ContentStore(), reader) if err != nil { - // is.Update() is not called on error return nil, err } - is := c.ImageService() - var images []Image - for _, imgrec := range imgrecs { - if updated, err := is.Update(ctx, imgrec, "target"); err != nil { + var ( + imgs []images.Image + cs = c.ContentStore() + is = c.ImageService() + ) + + if iopts.indexName != "" { + imgs = append(imgs, images.Image{ + Name: iopts.indexName, + Target: index, + }) + } + + var handler images.HandlerFunc + handler = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + // Only save images at top level + if desc.Digest != index.Digest { + return images.Children(ctx, cs, desc) + } + + p, err := content.ReadBlob(ctx, cs, desc) + if err != nil { + return nil, err + } + + var idx ocispec.Index + if err := json.Unmarshal(p, &idx); err != nil { + return nil, err + } + + for _, m := range idx.Manifests { + if ref := m.Annotations[ocispec.AnnotationRefName]; ref != "" { + if iopts.imageRefT != nil { + ref = iopts.imageRefT(ref) + } + if ref != "" { + imgs = append(imgs, images.Image{ + Name: ref, + Target: m, + }) + } + } + if iopts.dgstRefT != nil { + ref := iopts.dgstRefT(m.Digest) + if ref != "" { + imgs = append(imgs, images.Image{ + Name: ref, + Target: m, + }) + } + } + } + + return idx.Manifests, nil + } + + handler = images.SetChildrenLabels(cs, handler) + if err := images.Walk(ctx, handler, index); err != nil { + return nil, err + } + + for i := range imgs { + img, err := is.Update(ctx, imgs[i], "target") + if err != nil { if !errdefs.IsNotFound(err) { return nil, err } - created, err := is.Create(ctx, imgrec) + img, err = is.Create(ctx, imgs[i]) if err != nil { return nil, err } - - imgrec = created - } else { - imgrec = updated } - - images = append(images, NewImage(c, imgrec)) + imgs[i] = img } - return images, nil + + return imgs, nil } diff --git a/import_test.go b/import_test.go index 62813ac0c..10205baca 100644 --- a/import_test.go +++ b/import_test.go @@ -49,13 +49,17 @@ func TestOCIExportAndImport(t *testing.T) { t.Fatal(err) } - imgrecs, err := client.Import(ctx, &oci.V1Importer{ImageName: "foo/bar:"}, exported) + opts := []ImportOpt{ + WithImporter(&oci.V1Importer{}), + WithImageRefTranslator(oci.RefTranslator("foo/bar")), + } + imgrecs, err := client.Import(ctx, exported, opts...) if err != nil { t.Fatal(err) } for _, imgrec := range imgrecs { - err = client.ImageService().Delete(ctx, imgrec.Name()) + err = client.ImageService().Delete(ctx, imgrec.Name) if err != nil { t.Fatal(err) } From 9e6db7195485f70d0798e5aebdb7c96254812be1 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 11 Sep 2018 14:03:05 -0700 Subject: [PATCH 2/5] Add docker importer Update ctr to support all formats by default Signed-off-by: Derek McGowan --- cmd/ctr/commands/images/import.go | 49 ++-- images/docker/importer.go | 397 ++++++++++++++++++++++++++++++ 2 files changed, 429 insertions(+), 17 deletions(-) create mode 100644 images/docker/importer.go diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index b19131575..8b6bfcc2b 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -24,6 +24,7 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/cmd/ctr/commands" + "github.com/containerd/containerd/images/docker" oci "github.com/containerd/containerd/images/oci" "github.com/containerd/containerd/log" "github.com/urfave/cli" @@ -35,11 +36,14 @@ var importCommand = cli.Command{ ArgsUsage: "[flags] ", Description: `Import images from a tar stream. Implemented formats: -- oci.v1 (default) +- oci.v1 +- docker.v1.1 +- docker.v1.2 -For oci.v1 format, you need to specify --oci-name because an OCI archive contains image refs (tags) -but does not contain the base image name. +For OCI v1, you may need to specify --base-name because an OCI archive +contains only partial image references (tags without the base image name). +If no base image name is provided, a name will be generated as "import-%{date}". e.g. $ ctr images import --format oci.v1 --oci-name foo/bar foobar.tar @@ -50,18 +54,22 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb Flags: append([]cli.Flag{ cli.StringFlag{ Name: "format", - Value: "oci.v1", - Usage: "image format. See DESCRIPTION.", + Value: "", + Usage: "image format, by default supports OCI v1, Docker v1.1, Docker v1.2", }, cli.StringFlag{ - Name: "prefix,oci-name", + Name: "base-name,oci-name", Value: "", - Usage: "prefix image name for added images", + Usage: "base image name for added images, when provided images without this name prefix are filtered out", }, cli.BoolFlag{ Name: "digests", Usage: "whether to create digest images", }, + cli.StringFlag{ + Name: "index-name", + Usage: "image name to keep index as, by default index is discarded", + }, }, commands.SnapshotterFlags...), Action: func(context *cli.Context) error { @@ -70,23 +78,30 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb opts []containerd.ImportOpt ) + prefix := context.String("base-name") + if prefix == "" { + prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) + } + switch format := context.String("format"); format { - case "oci.v1": + case "", "docker", "docker.v1.1", "docker.v1.2": + opts = append(opts, containerd.WithImporter(&docker.V1Importer{})) + opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, context.String("base-name") != ""))) + case "oci", "oci.v1": opts = append(opts, containerd.WithImporter(&oci.V1Importer{})) - - prefix := context.String("prefix") - if prefix == "" { - prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) - } - opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix))) - if context.Bool("digests") { - opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix))) - } default: return fmt.Errorf("unknown format %s", format) } + if context.Bool("digests") { + opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix))) + } + + if idxName := context.String("index-name"); idxName != "" { + opts = append(opts, containerd.WithIndexName(idxName)) + } + client, ctx, cancel, err := commands.NewClient(context) if err != nil { return err diff --git a/images/docker/importer.go b/images/docker/importer.go new file mode 100644 index 000000000..8d46287f2 --- /dev/null +++ b/images/docker/importer.go @@ -0,0 +1,397 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package docker provides a Docker compatible importer capable of +// importing both Docker and OCI formats. +package docker + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "io" + "io/ioutil" + "path" + "strings" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/log" + "github.com/containerd/cri/pkg/util" + digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// V1Importer implements OCI Image Spec v1. +type V1Importer struct { + // TODO: Add option to compress layers on ingest +} + +var _ images.Importer = &V1Importer{} + +// Import implements Importer. +func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { + var ( + desc ocispec.Descriptor + tr = tar.NewReader(reader) + + mfsts []manifestDotJSON + symlinkLayers = make(map[string]string) // key: filename (foobar/layer.tar), value: linkname (targetlayerid/layer.tar) + layers = make(map[string]ocispec.Descriptor) // key: filename (foobar/layer.tar) + configs = make(map[string]imageConfig) // key: filename (foobar.json) + + ) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return ocispec.Descriptor{}, err + } + if hdr.Typeflag == tar.TypeSymlink && isLayerTar(hdr.Name) { + linkname, err := followSymlinkLayer(hdr.Linkname) + if err != nil { + log.G(ctx).WithError(err).WithField("file", hdr.Name).Debugf("symlink to %s ignored", hdr.Linkname) + } else { + symlinkLayers[hdr.Name] = linkname + } + continue + } + + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") + continue + } + hdrName := path.Clean(hdr.Name) + if hdrName == "index.json" { + if desc.Digest != "" { + return ocispec.Descriptor{}, errors.New("duplicated index.json") + } + desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size) + if err != nil { + return ocispec.Descriptor{}, err + } + } else if strings.HasPrefix(hdrName, "blobs/") { + if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { + return ocispec.Descriptor{}, err + } + } else if hdrName == ocispec.ImageLayoutFile { + // TODO Validate + } else if hdr.Name == "manifest.json" { + mfsts, err = onUntarManifestJSON(tr) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name) + } + } else if isLayerTar(hdr.Name) { + desc, err := onUntarLayerTar(ctx, tr, store, hdr.Name, hdr.Size) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "untar layer %q", hdr.Name) + } + layers[hdr.Name] = desc + } else if isDotJSON(hdr.Name) { + c, err := onUntarDotJSON(ctx, tr, store, hdr.Name, hdr.Size) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "untar config %q", hdr.Name) + } + configs[hdr.Name] = c + } else { + log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored") + } + + } + if desc.Digest != "" { + return desc, nil + } + + for name, linkname := range symlinkLayers { + desc, ok := layers[linkname] + if !ok { + return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname) + } + layers[name] = desc + } + + var idx ocispec.Index + for _, mfst := range mfsts { + config, ok := configs[mfst.Config] + if !ok { + return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config) + } + + layers, err := resolveLayers(mfst.Layers, layers) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers") + } + + manifest := ocispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: config.desc, + Layers: layers, + } + + desc, err := writeManifest(ctx, store, manifest, config.img.Architecture, config.img.OS) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest") + } + + if len(mfst.RepoTags) == 0 { + idx.Manifests = append(idx.Manifests, desc) + } else { + // Add descriptor per tag + for _, ref := range mfst.RepoTags { + msftdesc := desc + + // TODO: Replace this function to not depend on reference package + normalized, err := util.NormalizeImageRef(ref) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "normalize image ref %q", ref) + } + + msftdesc.Annotations = map[string]string{ + ocispec.AnnotationRefName: normalized.String(), + } + + idx.Manifests = append(idx.Manifests, msftdesc) + } + } + } + + return writeIndex(ctx, store, idx) +} + +// RefTranslator creates a reference which only has a tag or verifies +// a full reference. +func RefTranslator(image string, checkPrefix bool) func(string) string { + return func(ref string) string { + // Check if ref is full reference + if strings.ContainsAny(ref, "/:@") { + // If not prefixed, don't include image + if checkPrefix && !isImagePrefix(ref, image) { + return "" + } + return ref + } + return image + ":" + ref + } +} + +func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) { + b, err := ioutil.ReadAll(r) + if err != nil { + return ocispec.Descriptor{}, err + } + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageIndex, + Digest: digest.FromBytes(b), + Size: size, + } + if int64(len(b)) != size { + return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size) + } + + if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil { + return ocispec.Descriptor{}, err + } + + return desc, err +} + +func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error { + // name is like "blobs/sha256/deadbeef" + split := strings.Split(name, "/") + if len(split) != 3 { + return errors.Errorf("unexpected name: %q", name) + } + algo := digest.Algorithm(split[1]) + if !algo.Available() { + return errors.Errorf("unsupported algorithm: %s", algo) + } + dgst := digest.NewDigestFromHex(algo.String(), split[2]) + return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) +} + +// manifestDotJSON is an entry in manifest.json. +type manifestDotJSON struct { + Config string + RepoTags []string + Layers []string + // Parent is unsupported + Parent string +} + +// isLayerTar returns true if name is like "foobar/layer.tar" +func isLayerTar(name string) bool { + slashes := len(strings.Split(name, "/")) + return slashes == 2 && strings.HasSuffix(name, "/layer.tar") +} + +// followSymlinkLayer returns actual layer name of the symlink layer. +// It returns "foobar/layer.tar" if the name is like +// "../foobar/layer.tar", and returns error if the name +// is not in "../foobar/layer.tar" format. +func followSymlinkLayer(name string) (string, error) { + parts := strings.Split(name, "/") + if len(parts) != 3 || parts[0] != ".." { + return "", errors.New("invalid symlink layer") + } + name = strings.TrimPrefix(name, "../") + if !isLayerTar(name) { + return "", errors.New("invalid layer tar") + } + return name, nil +} + +// isDotJSON returns true if name is like "foobar.json" +func isDotJSON(name string) bool { + slashes := len(strings.Split(name, "/")) + return slashes == 1 && strings.HasSuffix(name, ".json") +} + +func resolveLayers(layerIDs []string, layerIDMap map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) { + var layers []ocispec.Descriptor + for _, f := range layerIDs { + desc, ok := layerIDMap[f] + if !ok { + return nil, errors.Errorf("layer %q not found", f) + } + layers = append(layers, desc) + } + return layers, nil +} + +func writeManifest(ctx context.Context, cs content.Ingester, manifest ocispec.Manifest, arch, os string) (ocispec.Descriptor, error) { + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return ocispec.Descriptor{}, err + } + + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.FromBytes(manifestBytes), + Size: int64(len(manifestBytes)), + } + if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil { + return ocispec.Descriptor{}, err + } + + if arch != "" || os != "" { + desc.Platform = &ocispec.Platform{ + Architecture: arch, + OS: os, + } + } + return desc, nil +} + +func writeIndex(ctx context.Context, cs content.Ingester, manifest ocispec.Index) (ocispec.Descriptor, error) { + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return ocispec.Descriptor{}, err + } + + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageIndex, + Digest: digest.FromBytes(manifestBytes), + Size: int64(len(manifestBytes)), + } + if err := content.WriteBlob(ctx, cs, "index-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil { + return ocispec.Descriptor{}, err + } + + return desc, nil +} + +func onUntarManifestJSON(r io.Reader) ([]manifestDotJSON, error) { + // name: "manifest.json" + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + var mfsts []manifestDotJSON + if err := json.Unmarshal(b, &mfsts); err != nil { + return nil, err + } + return mfsts, nil +} + +func onUntarLayerTar(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (ocispec.Descriptor, error) { + // name is like "foobar/layer.tar" ( guaranteed by isLayerTar() ) + split := strings.Split(name, "/") + // note: split[0] is not expected digest here + cw, err := cs.Writer(ctx, content.WithRef("layer-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size})) + if err != nil { + return ocispec.Descriptor{}, err + } + // TODO: support compression and committing with labels + defer cw.Close() + if err := content.Copy(ctx, cw, r, size, ""); err != nil { + return ocispec.Descriptor{}, err + } + return ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageLayer, + Size: size, + Digest: cw.Digest(), + }, nil +} + +type imageConfig struct { + desc ocispec.Descriptor + img ocispec.Image +} + +func onUntarDotJSON(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (imageConfig, error) { + config := imageConfig{} + config.desc.MediaType = ocispec.MediaTypeImageConfig + config.desc.Size = size + // name is like "foobar.json" ( guaranteed by is DotJSON() ) + split := strings.Split(name, ".") + cw, err := cs.Writer(ctx, content.WithRef("config-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size})) + if err != nil { + return imageConfig{}, err + } + defer cw.Close() + var buf bytes.Buffer + tr := io.TeeReader(r, &buf) + if err := content.Copy(ctx, cw, tr, size, ""); err != nil { + return imageConfig{}, err + } + config.desc.Digest = cw.Digest() + if err := json.Unmarshal(buf.Bytes(), &config.img); err != nil { + return imageConfig{}, err + } + return config, nil +} + +func isImagePrefix(s, prefix string) bool { + if !strings.HasPrefix(s, prefix) { + return false + } + if len(s) > len(prefix) { + switch s[len(prefix)] { + case '/', ':', '@': + // Prevent matching partial namespaces + default: + return false + } + } + return true +} From bce20b75da6264f5f26e26130e9a0d52c006ecfd Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 11 Sep 2018 18:34:25 -0700 Subject: [PATCH 3/5] Simplify docker importer Support any layout and rely on manifest.json to reference blobs Signed-off-by: Derek McGowan --- cmd/ctr/commands/images/import.go | 11 +- images/docker/importer.go | 351 +++++++++--------------------- images/docker/reference.go | 65 ++++++ images/oci/importer.go | 9 - import_test.go | 3 +- 5 files changed, 179 insertions(+), 260 deletions(-) create mode 100644 images/docker/reference.go diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index 8b6bfcc2b..eab50d215 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "os" + "strings" "time" "github.com/containerd/containerd" @@ -81,15 +82,19 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb prefix := context.String("base-name") if prefix == "" { prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) + opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, false))) + } else { + // When provided, filter out references which do not match + opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, true))) } switch format := context.String("format"); format { case "", "docker", "docker.v1.1", "docker.v1.2": - opts = append(opts, containerd.WithImporter(&docker.V1Importer{})) - opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, context.String("base-name") != ""))) + opts = append(opts, containerd.WithImporter(&docker.V1Importer{ + SkipOCI: strings.HasPrefix(format, "docker"), + })) case "oci", "oci.v1": opts = append(opts, containerd.WithImporter(&oci.V1Importer{})) - opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix))) default: return fmt.Errorf("unknown format %s", format) } diff --git a/images/docker/importer.go b/images/docker/importer.go index 8d46287f2..5ddfa81d4 100644 --- a/images/docker/importer.go +++ b/images/docker/importer.go @@ -26,21 +26,24 @@ import ( "io" "io/ioutil" "path" - "strings" + "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" - "github.com/containerd/cri/pkg/util" digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) -// V1Importer implements OCI Image Spec v1. +// V1Importer implements Docker v1.1, v1.2 and OCI v1. type V1Importer struct { + // SkipOCI prevent interpretting OCI files + SkipOCI bool + // TODO: Add option to compress layers on ingest + } var _ images.Importer = &V1Importer{} @@ -48,14 +51,16 @@ var _ images.Importer = &V1Importer{} // Import implements Importer. func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { var ( - desc ocispec.Descriptor - tr = tar.NewReader(reader) - - mfsts []manifestDotJSON - symlinkLayers = make(map[string]string) // key: filename (foobar/layer.tar), value: linkname (targetlayerid/layer.tar) - layers = make(map[string]ocispec.Descriptor) // key: filename (foobar/layer.tar) - configs = make(map[string]imageConfig) // key: filename (foobar.json) + tr = tar.NewReader(reader) + ociLayout ocispec.ImageLayout + mfsts []struct { + Config string + RepoTags []string + Layers []string + } + symlinks = make(map[string]string) + blobs = make(map[string]ocispec.Descriptor) ) for { hdr, err := tr.Next() @@ -65,77 +70,70 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io if err != nil { return ocispec.Descriptor{}, err } - if hdr.Typeflag == tar.TypeSymlink && isLayerTar(hdr.Name) { - linkname, err := followSymlinkLayer(hdr.Linkname) - if err != nil { - log.G(ctx).WithError(err).WithField("file", hdr.Name).Debugf("symlink to %s ignored", hdr.Linkname) - } else { - symlinkLayers[hdr.Name] = linkname - } - continue + if hdr.Typeflag == tar.TypeSymlink { + symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname) } if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { - log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") + if hdr.Typeflag != tar.TypeDir { + log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") + } continue } + hdrName := path.Clean(hdr.Name) - if hdrName == "index.json" { - if desc.Digest != "" { - return ocispec.Descriptor{}, errors.New("duplicated index.json") + if hdrName == ocispec.ImageLayoutFile && !oi.SkipOCI { + if err = onUntarJSON(tr, &ociLayout); err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name) } - desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size) - if err != nil { - return ocispec.Descriptor{}, err - } - } else if strings.HasPrefix(hdrName, "blobs/") { - if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { - return ocispec.Descriptor{}, err - } - } else if hdrName == ocispec.ImageLayoutFile { - // TODO Validate - } else if hdr.Name == "manifest.json" { - mfsts, err = onUntarManifestJSON(tr) - if err != nil { + } else if hdrName == "manifest.json" { + if err = onUntarJSON(tr, &mfsts); err != nil { return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name) } - } else if isLayerTar(hdr.Name) { - desc, err := onUntarLayerTar(ctx, tr, store, hdr.Name, hdr.Size) - if err != nil { - return ocispec.Descriptor{}, errors.Wrapf(err, "untar layer %q", hdr.Name) - } - layers[hdr.Name] = desc - } else if isDotJSON(hdr.Name) { - c, err := onUntarDotJSON(ctx, tr, store, hdr.Name, hdr.Size) - if err != nil { - return ocispec.Descriptor{}, errors.Wrapf(err, "untar config %q", hdr.Name) - } - configs[hdr.Name] = c } else { - log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored") + dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "failed to ingest %q", hdr.Name) + } + + blobs[hdrName] = ocispec.Descriptor{ + Digest: dgst, + Size: hdr.Size, + } + } + } + + if ociLayout.Version != "" { + if ociLayout.Version != ocispec.ImageLayoutVersion { + return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version) } - } - if desc.Digest != "" { - return desc, nil + idx, ok := blobs["index.json"] + if !ok { + return ocispec.Descriptor{}, errors.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion) + } + + idx.MediaType = ocispec.MediaTypeImageIndex + return idx, nil } - for name, linkname := range symlinkLayers { - desc, ok := layers[linkname] + for name, linkname := range symlinks { + desc, ok := blobs[linkname] if !ok { return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname) } - layers[name] = desc + blobs[name] = desc } var idx ocispec.Index for _, mfst := range mfsts { - config, ok := configs[mfst.Config] + config, ok := blobs[mfst.Config] if !ok { return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config) } + config.MediaType = ocispec.MediaTypeImageConfig - layers, err := resolveLayers(mfst.Layers, layers) + layers, err := resolveLayers(ctx, store, mfst.Layers, blobs) if err != nil { return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers") } @@ -144,15 +142,24 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io Versioned: specs.Versioned{ SchemaVersion: 2, }, - Config: config.desc, + Config: config, Layers: layers, } - desc, err := writeManifest(ctx, store, manifest, config.img.Architecture, config.img.OS) + desc, err := writeManifest(ctx, store, manifest, ocispec.MediaTypeImageManifest) if err != nil { return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest") } + platforms, err := images.Platforms(ctx, store, desc) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform") + } + if len(platforms) > 0 { + // Only one platform can be resolved from non-index manifest + desc.Platform = &platforms[0] + } + if len(mfst.RepoTags) == 0 { idx.Manifests = append(idx.Manifests, desc) } else { @@ -160,14 +167,13 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io for _, ref := range mfst.RepoTags { msftdesc := desc - // TODO: Replace this function to not depend on reference package - normalized, err := util.NormalizeImageRef(ref) + normalized, err := normalizeReference(ref) if err != nil { - return ocispec.Descriptor{}, errors.Wrapf(err, "normalize image ref %q", ref) + return ocispec.Descriptor{}, err } msftdesc.Annotations = map[string]string{ - ocispec.AnnotationRefName: normalized.String(), + ocispec.AnnotationRefName: normalized, } idx.Manifests = append(idx.Manifests, msftdesc) @@ -175,117 +181,68 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io } } - return writeIndex(ctx, store, idx) + return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex) } -// RefTranslator creates a reference which only has a tag or verifies -// a full reference. -func RefTranslator(image string, checkPrefix bool) func(string) string { - return func(ref string) string { - // Check if ref is full reference - if strings.ContainsAny(ref, "/:@") { - // If not prefixed, don't include image - if checkPrefix && !isImagePrefix(ref, image) { - return "" - } - return ref - } - return image + ":" + ref - } -} - -func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) { +func onUntarJSON(r io.Reader, j interface{}) error { b, err := ioutil.ReadAll(r) if err != nil { - return ocispec.Descriptor{}, err + return err } - desc := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageIndex, - Digest: digest.FromBytes(b), - Size: size, + if err := json.Unmarshal(b, j); err != nil { + return err } - if int64(len(b)) != size { - return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size) - } - - if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil { - return ocispec.Descriptor{}, err - } - - return desc, err + return nil } -func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error { - // name is like "blobs/sha256/deadbeef" - split := strings.Split(name, "/") - if len(split) != 3 { - return errors.Errorf("unexpected name: %q", name) +func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) { + dgstr := digest.Canonical.Digester() + + if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil { + return "", err } - algo := digest.Algorithm(split[1]) - if !algo.Available() { - return errors.Errorf("unsupported algorithm: %s", algo) - } - dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) + + return dgstr.Digest(), nil } -// manifestDotJSON is an entry in manifest.json. -type manifestDotJSON struct { - Config string - RepoTags []string - Layers []string - // Parent is unsupported - Parent string -} - -// isLayerTar returns true if name is like "foobar/layer.tar" -func isLayerTar(name string) bool { - slashes := len(strings.Split(name, "/")) - return slashes == 2 && strings.HasSuffix(name, "/layer.tar") -} - -// followSymlinkLayer returns actual layer name of the symlink layer. -// It returns "foobar/layer.tar" if the name is like -// "../foobar/layer.tar", and returns error if the name -// is not in "../foobar/layer.tar" format. -func followSymlinkLayer(name string) (string, error) { - parts := strings.Split(name, "/") - if len(parts) != 3 || parts[0] != ".." { - return "", errors.New("invalid symlink layer") - } - name = strings.TrimPrefix(name, "../") - if !isLayerTar(name) { - return "", errors.New("invalid layer tar") - } - return name, nil -} - -// isDotJSON returns true if name is like "foobar.json" -func isDotJSON(name string) bool { - slashes := len(strings.Split(name, "/")) - return slashes == 1 && strings.HasSuffix(name, ".json") -} - -func resolveLayers(layerIDs []string, layerIDMap map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) { +func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) { var layers []ocispec.Descriptor - for _, f := range layerIDs { - desc, ok := layerIDMap[f] + for _, f := range layerFiles { + desc, ok := blobs[f] if !ok { return nil, errors.Errorf("layer %q not found", f) } + + // Open blob, resolve media type + ra, err := store.ReaderAt(ctx, desc) + if err != nil { + return nil, errors.Wrapf(err, "failed to open %q (%s)", f, desc.Digest) + } + s, err := compression.DecompressStream(content.NewReader(ra)) + if err != nil { + return nil, errors.Wrapf(err, "failed to detect compression for %q", f) + } + if s.GetCompression() == compression.Uncompressed { + // TODO: Support compressing and writing back to content store + desc.MediaType = ocispec.MediaTypeImageLayer + } else { + desc.MediaType = ocispec.MediaTypeImageLayerGzip + } + s.Close() + layers = append(layers, desc) } return layers, nil } -func writeManifest(ctx context.Context, cs content.Ingester, manifest ocispec.Manifest, arch, os string) (ocispec.Descriptor, error) { +func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) { manifestBytes, err := json.Marshal(manifest) if err != nil { return ocispec.Descriptor{}, err } desc := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageManifest, + MediaType: mediaType, Digest: digest.FromBytes(manifestBytes), Size: int64(len(manifestBytes)), } @@ -293,105 +250,5 @@ func writeManifest(ctx context.Context, cs content.Ingester, manifest ocispec.Ma return ocispec.Descriptor{}, err } - if arch != "" || os != "" { - desc.Platform = &ocispec.Platform{ - Architecture: arch, - OS: os, - } - } return desc, nil } - -func writeIndex(ctx context.Context, cs content.Ingester, manifest ocispec.Index) (ocispec.Descriptor, error) { - manifestBytes, err := json.Marshal(manifest) - if err != nil { - return ocispec.Descriptor{}, err - } - - desc := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageIndex, - Digest: digest.FromBytes(manifestBytes), - Size: int64(len(manifestBytes)), - } - if err := content.WriteBlob(ctx, cs, "index-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil { - return ocispec.Descriptor{}, err - } - - return desc, nil -} - -func onUntarManifestJSON(r io.Reader) ([]manifestDotJSON, error) { - // name: "manifest.json" - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - var mfsts []manifestDotJSON - if err := json.Unmarshal(b, &mfsts); err != nil { - return nil, err - } - return mfsts, nil -} - -func onUntarLayerTar(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (ocispec.Descriptor, error) { - // name is like "foobar/layer.tar" ( guaranteed by isLayerTar() ) - split := strings.Split(name, "/") - // note: split[0] is not expected digest here - cw, err := cs.Writer(ctx, content.WithRef("layer-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size})) - if err != nil { - return ocispec.Descriptor{}, err - } - // TODO: support compression and committing with labels - defer cw.Close() - if err := content.Copy(ctx, cw, r, size, ""); err != nil { - return ocispec.Descriptor{}, err - } - return ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageLayer, - Size: size, - Digest: cw.Digest(), - }, nil -} - -type imageConfig struct { - desc ocispec.Descriptor - img ocispec.Image -} - -func onUntarDotJSON(ctx context.Context, r io.Reader, cs content.Ingester, name string, size int64) (imageConfig, error) { - config := imageConfig{} - config.desc.MediaType = ocispec.MediaTypeImageConfig - config.desc.Size = size - // name is like "foobar.json" ( guaranteed by is DotJSON() ) - split := strings.Split(name, ".") - cw, err := cs.Writer(ctx, content.WithRef("config-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size})) - if err != nil { - return imageConfig{}, err - } - defer cw.Close() - var buf bytes.Buffer - tr := io.TeeReader(r, &buf) - if err := content.Copy(ctx, cw, tr, size, ""); err != nil { - return imageConfig{}, err - } - config.desc.Digest = cw.Digest() - if err := json.Unmarshal(buf.Bytes(), &config.img); err != nil { - return imageConfig{}, err - } - return config, nil -} - -func isImagePrefix(s, prefix string) bool { - if !strings.HasPrefix(s, prefix) { - return false - } - if len(s) > len(prefix) { - switch s[len(prefix)] { - case '/', ':', '@': - // Prevent matching partial namespaces - default: - return false - } - } - return true -} diff --git a/images/docker/reference.go b/images/docker/reference.go new file mode 100644 index 000000000..ed7d7d91f --- /dev/null +++ b/images/docker/reference.go @@ -0,0 +1,65 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package docker + +import ( + "strings" + + "github.com/containerd/cri/pkg/util" + "github.com/pkg/errors" +) + +// RefTranslator creates a reference which only has a tag or verifies +// a full reference. +func RefTranslator(image string, checkPrefix bool) func(string) string { + return func(ref string) string { + // Check if ref is full reference + if strings.ContainsAny(ref, "/:@") { + // If not prefixed, don't include image + if checkPrefix && !isImagePrefix(ref, image) { + return "" + } + return ref + } + return image + ":" + ref + } +} + +func isImagePrefix(s, prefix string) bool { + if !strings.HasPrefix(s, prefix) { + return false + } + if len(s) > len(prefix) { + switch s[len(prefix)] { + case '/', ':', '@': + // Prevent matching partial namespaces + default: + return false + } + } + return true +} + +func normalizeReference(ref string) (string, error) { + // TODO: Replace this function to not depend on reference package + normalized, err := util.NormalizeImageRef(ref) + if err != nil { + return "", errors.Wrapf(err, "normalize image ref %q", ref) + } + + return normalized.String(), nil +} diff --git a/images/oci/importer.go b/images/oci/importer.go index 391d5b9f3..6c2b9803f 100644 --- a/images/oci/importer.go +++ b/images/oci/importer.go @@ -118,15 +118,6 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) } -// RefTranslator creates a reference using an OCI ref annotation, -// which is mentioned in the spec as only a tag compontent, -// concatenated with an image name -func RefTranslator(prefix string) func(string) string { - return func(ref string) string { - return prefix + ":" + ref - } -} - // DigestTranslator creates a digest reference by adding the // digest to an image name func DigestTranslator(prefix string) func(digest.Digest) string { diff --git a/import_test.go b/import_test.go index 10205baca..d72d47eb2 100644 --- a/import_test.go +++ b/import_test.go @@ -20,6 +20,7 @@ import ( "runtime" "testing" + "github.com/containerd/containerd/images/docker" "github.com/containerd/containerd/images/oci" ) @@ -51,7 +52,7 @@ func TestOCIExportAndImport(t *testing.T) { opts := []ImportOpt{ WithImporter(&oci.V1Importer{}), - WithImageRefTranslator(oci.RefTranslator("foo/bar")), + WithImageRefTranslator(docker.RefTranslator("foo/bar", false)), } imgrecs, err := client.Import(ctx, exported, opts...) if err != nil { From a62be324b76f6d5f415d46963df26ee090aa34a7 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 12 Sep 2018 16:58:22 -0700 Subject: [PATCH 4/5] Unify docker and oci importer Signed-off-by: Derek McGowan --- cmd/ctr/commands/images/import.go | 40 ++------ content/helpers.go | 6 +- images/{docker => archive}/importer.go | 42 ++++---- images/{docker => archive}/reference.go | 27 ++++- images/oci/importer.go | 127 ------------------------ import.go | 18 +--- import_test.go | 7 +- 7 files changed, 64 insertions(+), 203 deletions(-) rename images/{docker => archive}/importer.go (85%) rename images/{docker => archive}/reference.go (63%) delete mode 100644 images/oci/importer.go diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index eab50d215..2711ad0ee 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -20,13 +20,11 @@ import ( "fmt" "io" "os" - "strings" "time" "github.com/containerd/containerd" "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/images/docker" - oci "github.com/containerd/containerd/images/oci" + "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/log" "github.com/urfave/cli" ) @@ -42,30 +40,25 @@ Implemented formats: - docker.v1.2 -For OCI v1, you may need to specify --base-name because an OCI archive -contains only partial image references (tags without the base image name). -If no base image name is provided, a name will be generated as "import-%{date}". +For OCI v1, you may need to specify --base-name because an OCI archive may +contain only partial image references (tags without the base image name). +If no base image name is provided, a name will be generated as "import-%{yyyy-MM-dd}". e.g. - $ ctr images import --format oci.v1 --oci-name foo/bar foobar.tar + $ ctr images import --base-name foo/bar foobar.tar If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadbeef", the command will create "foo/bar:latest" and "foo/bar@sha256:deadbeef" images in the containerd store. `, Flags: append([]cli.Flag{ cli.StringFlag{ - Name: "format", + Name: "base-name", Value: "", - Usage: "image format, by default supports OCI v1, Docker v1.1, Docker v1.2", - }, - cli.StringFlag{ - Name: "base-name,oci-name", - Value: "", - Usage: "base image name for added images, when provided images without this name prefix are filtered out", + Usage: "base image name for added images, when provided only images with this name prefix are imported", }, cli.BoolFlag{ Name: "digests", - Usage: "whether to create digest images", + Usage: "whether to create digest images (default: false)", }, cli.StringFlag{ Name: "index-name", @@ -82,25 +75,14 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb prefix := context.String("base-name") if prefix == "" { prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) - opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, false))) + opts = append(opts, containerd.WithImageRefTranslator(archive.AddRefPrefix(prefix))) } else { // When provided, filter out references which do not match - opts = append(opts, containerd.WithImageRefTranslator(docker.RefTranslator(prefix, true))) - } - - switch format := context.String("format"); format { - case "", "docker", "docker.v1.1", "docker.v1.2": - opts = append(opts, containerd.WithImporter(&docker.V1Importer{ - SkipOCI: strings.HasPrefix(format, "docker"), - })) - case "oci", "oci.v1": - opts = append(opts, containerd.WithImporter(&oci.V1Importer{})) - default: - return fmt.Errorf("unknown format %s", format) + opts = append(opts, containerd.WithImageRefTranslator(archive.FilterRefPrefix(prefix))) } if context.Bool("digests") { - opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix))) + opts = append(opts, containerd.WithDigestRef(archive.DigestTranslator(prefix))) } if idxName := context.String("index-name"); idxName != "" { diff --git a/content/helpers.go b/content/helpers.go index 819b7ea1e..3e231408d 100644 --- a/content/helpers.go +++ b/content/helpers.go @@ -70,7 +70,7 @@ func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, desc o cw, err := OpenWriter(ctx, cs, WithRef(ref), WithDescriptor(desc)) if err != nil { if !errdefs.IsAlreadyExists(err) { - return err + return errors.Wrap(err, "failed to open writer") } return nil // all ready present @@ -127,7 +127,7 @@ func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, er func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error { ws, err := cw.Status() if err != nil { - return err + return errors.Wrap(err, "failed to get status") } if ws.Offset > 0 { @@ -138,7 +138,7 @@ func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected dige } if _, err := copyWithBuffer(cw, r); err != nil { - return err + return errors.Wrap(err, "failed to copy") } if err := cw.Commit(ctx, size, expected, opts...); err != nil { diff --git a/images/docker/importer.go b/images/archive/importer.go similarity index 85% rename from images/docker/importer.go rename to images/archive/importer.go index 5ddfa81d4..59514c66a 100644 --- a/images/docker/importer.go +++ b/images/archive/importer.go @@ -14,9 +14,8 @@ limitations under the License. */ -// Package docker provides a Docker compatible importer capable of -// importing both Docker and OCI formats. -package docker +// Package archive provides a Docker and OCI compatible importer +package archive import ( "archive/tar" @@ -37,19 +36,15 @@ import ( "github.com/pkg/errors" ) -// V1Importer implements Docker v1.1, v1.2 and OCI v1. -type V1Importer struct { - // SkipOCI prevent interpretting OCI files - SkipOCI bool - - // TODO: Add option to compress layers on ingest - -} - -var _ images.Importer = &V1Importer{} - -// Import implements Importer. -func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { +// ImportIndex imports an index from a tar achive image bundle +// - implements Docker v1.1, v1.2 and OCI v1. +// - prefers OCI v1 when provided +// - creates OCI index for Docker formats +// - normalizes Docker references and adds as OCI ref name +// e.g. alpine:latest -> docker.io/library/alpine:latest +// - existing OCI reference names are untouched +// - TODO: support option to compress layers on ingest +func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { var ( tr = tar.NewReader(reader) @@ -82,7 +77,7 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io } hdrName := path.Clean(hdr.Name) - if hdrName == ocispec.ImageLayoutFile && !oi.SkipOCI { + if hdrName == ocispec.ImageLayoutFile { if err = onUntarJSON(tr, &ociLayout); err != nil { return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name) } @@ -103,6 +98,9 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io } } + // If OCI layout was given, interpret the tar as an OCI layout. + // When not provided, the layout of the tar will be interpretted + // as Docker v1.1 or v1.2. if ociLayout.Version != "" { if ociLayout.Version != ocispec.ImageLayoutVersion { return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version) @@ -156,7 +154,9 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform") } if len(platforms) > 0 { - // Only one platform can be resolved from non-index manifest + // Only one platform can be resolved from non-index manifest, + // The platform can only come from the config included above, + // if the config has no platform it can be safely ommitted. desc.Platform = &platforms[0] } @@ -165,18 +165,18 @@ func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io } else { // Add descriptor per tag for _, ref := range mfst.RepoTags { - msftdesc := desc + mfstdesc := desc normalized, err := normalizeReference(ref) if err != nil { return ocispec.Descriptor{}, err } - msftdesc.Annotations = map[string]string{ + mfstdesc.Annotations = map[string]string{ ocispec.AnnotationRefName: normalized, } - idx.Manifests = append(idx.Manifests, msftdesc) + idx.Manifests = append(idx.Manifests, mfstdesc) } } } diff --git a/images/docker/reference.go b/images/archive/reference.go similarity index 63% rename from images/docker/reference.go rename to images/archive/reference.go index ed7d7d91f..0b1310181 100644 --- a/images/docker/reference.go +++ b/images/archive/reference.go @@ -14,18 +14,31 @@ limitations under the License. */ -package docker +package archive import ( "strings" "github.com/containerd/cri/pkg/util" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) -// RefTranslator creates a reference which only has a tag or verifies +// FilterRefPrefix restricts references to having the given image +// prefix. Tag-only references will have the prefix prepended. +func FilterRefPrefix(image string) func(string) string { + return refTranslator(image, true) +} + +// AddRefPrefix prepends the given image prefix to tag-only references, +// while leaving returning full references unmodified. +func AddRefPrefix(image string) func(string) string { + return refTranslator(image, false) +} + +// refTranslator creates a reference which only has a tag or verifies // a full reference. -func RefTranslator(image string, checkPrefix bool) func(string) string { +func refTranslator(image string, checkPrefix bool) func(string) string { return func(ref string) string { // Check if ref is full reference if strings.ContainsAny(ref, "/:@") { @@ -63,3 +76,11 @@ func normalizeReference(ref string) (string, error) { return normalized.String(), nil } + +// DigestTranslator creates a digest reference by adding the +// digest to an image name +func DigestTranslator(prefix string) func(digest.Digest) string { + return func(dgst digest.Digest) string { + return prefix + "@" + dgst.String() + } +} diff --git a/images/oci/importer.go b/images/oci/importer.go deleted file mode 100644 index 6c2b9803f..000000000 --- a/images/oci/importer.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package oci provides the importer and the exporter for OCI Image Spec. -package oci - -import ( - "archive/tar" - "bytes" - "context" - "io" - "io/ioutil" - "path" - "strings" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" - digest "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" -) - -// V1Importer implements OCI Image Spec v1. -type V1Importer struct{} - -var _ images.Importer = &V1Importer{} - -// Import implements Importer. -func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { - var ( - desc ocispec.Descriptor - tr = tar.NewReader(reader) - ) - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return ocispec.Descriptor{}, err - } - if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { - log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") - continue - } - hdrName := path.Clean(hdr.Name) - if hdrName == "index.json" { - if desc.Digest != "" { - return ocispec.Descriptor{}, errors.New("duplicated index.json") - } - desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size) - if err != nil { - return ocispec.Descriptor{}, err - } - } else if strings.HasPrefix(hdrName, "blobs/") { - if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { - return ocispec.Descriptor{}, err - } - } else if hdrName == ocispec.ImageLayoutFile { - // TODO Validate - } else { - log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored") - } - } - if desc.Digest == "" { - return ocispec.Descriptor{}, errors.New("no index.json found") - } - - return desc, nil -} - -func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) { - b, err := ioutil.ReadAll(r) - if err != nil { - return ocispec.Descriptor{}, err - } - desc := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageIndex, - Digest: digest.FromBytes(b), - Size: size, - } - if int64(len(b)) != size { - return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size) - } - - if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil { - return ocispec.Descriptor{}, err - } - - return desc, err -} - -func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error { - // name is like "blobs/sha256/deadbeef" - split := strings.Split(name, "/") - if len(split) != 3 { - return errors.Errorf("unexpected name: %q", name) - } - algo := digest.Algorithm(split[1]) - if !algo.Available() { - return errors.Errorf("unsupported algorithm: %s", algo) - } - dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) -} - -// DigestTranslator creates a digest reference by adding the -// digest to an image name -func DigestTranslator(prefix string) func(digest.Digest) string { - return func(dgst digest.Digest) string { - return prefix + "@" + dgst.String() - } -} diff --git a/import.go b/import.go index 4e42a7429..365056824 100644 --- a/import.go +++ b/import.go @@ -24,7 +24,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/images/oci" + "github.com/containerd/containerd/images/archive" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -33,7 +33,6 @@ type importOpts struct { indexName string imageRefT func(string) string dgstRefT func(digest.Digest) string - importer images.Importer } // ImportOpt allows the caller to specify import specific options @@ -65,15 +64,6 @@ func WithIndexName(name string) ImportOpt { } } -// WithImporter sets the importer to use for converting -// the read stream into an OCI Index. -func WithImporter(importer images.Importer) ImportOpt { - return func(c *importOpts) error { - c.importer = importer - return nil - } -} - // Import imports an image from a Tar stream using reader. // Caller needs to specify importer. Future version may use oci.v1 as the default. // Note that unreferrenced blobs may be imported to the content store as well. @@ -85,17 +75,13 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt } } - if iopts.importer == nil { - iopts.importer = &oci.V1Importer{} - } - ctx, done, err := c.WithLease(ctx) if err != nil { return nil, err } defer done(ctx) - index, err := iopts.importer.Import(ctx, c.ContentStore(), reader) + index, err := archive.ImportIndex(ctx, c.ContentStore(), reader) if err != nil { return nil, err } diff --git a/import_test.go b/import_test.go index d72d47eb2..817c7b296 100644 --- a/import_test.go +++ b/import_test.go @@ -20,7 +20,7 @@ import ( "runtime" "testing" - "github.com/containerd/containerd/images/docker" + "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/images/oci" ) @@ -51,12 +51,11 @@ func TestOCIExportAndImport(t *testing.T) { } opts := []ImportOpt{ - WithImporter(&oci.V1Importer{}), - WithImageRefTranslator(docker.RefTranslator("foo/bar", false)), + WithImageRefTranslator(archive.AddRefPrefix("foo/bar")), } imgrecs, err := client.Import(ctx, exported, opts...) if err != nil { - t.Fatal(err) + t.Fatalf("Import failed: %+v", err) } for _, imgrec := range imgrecs { From da6d29033c2a09ec3647353d034cc954ab04ffc5 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 12 Sep 2018 19:39:20 -0700 Subject: [PATCH 5/5] Clean up error messages Signed-off-by: Derek McGowan --- content/proxy/content_writer.go | 6 +++--- export.go | 3 ++- images/oci/exporter.go | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/content/proxy/content_writer.go b/content/proxy/content_writer.go index 6d35ba61a..5434a1568 100644 --- a/content/proxy/content_writer.go +++ b/content/proxy/content_writer.go @@ -57,7 +57,7 @@ func (rw *remoteWriter) Status() (content.Status, error) { Action: contentapi.WriteActionStat, }) if err != nil { - return content.Status{}, errors.Wrap(err, "error getting writer status") + return content.Status{}, errors.Wrap(errdefs.FromGRPC(err), "error getting writer status") } return content.Status{ @@ -82,7 +82,7 @@ func (rw *remoteWriter) Write(p []byte) (n int, err error) { Data: p, }) if err != nil { - return 0, err + return 0, errors.Wrap(errdefs.FromGRPC(err), "failed to send write") } n = int(resp.Offset - offset) @@ -112,7 +112,7 @@ func (rw *remoteWriter) Commit(ctx context.Context, size int64, expected digest. Labels: base.Labels, }) if err != nil { - return errdefs.FromGRPC(err) + return errors.Wrap(errdefs.FromGRPC(err), "commit failed") } if size != 0 && resp.Offset != size { diff --git a/export.go b/export.go index 7aac309ba..bfc25316c 100644 --- a/export.go +++ b/export.go @@ -22,6 +22,7 @@ import ( "github.com/containerd/containerd/images" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) type exportOpts struct { @@ -51,7 +52,7 @@ func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocis } pr, pw := io.Pipe() go func() { - pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw)) + pw.CloseWithError(errors.Wrap(exporter.Export(ctx, c.ContentStore(), desc, pw), "export failed")) }() return pr, nil } diff --git a/images/oci/exporter.go b/images/oci/exporter.go index d05c22b64..bf5751e6a 100644 --- a/images/oci/exporter.go +++ b/images/oci/exporter.go @@ -94,7 +94,7 @@ func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord { CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { r, err := cs.ReaderAt(ctx, desc) if err != nil { - return 0, err + return 0, errors.Wrap(err, "failed to get reader") } defer r.Close() @@ -103,7 +103,7 @@ func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord { n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) if err != nil { - return 0, err + return 0, errors.Wrap(err, "failed to copy to tar") } if dgstr.Digest() != desc.Digest { return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest())