diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index 37c3ab5c7..b19131575 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -20,9 +20,10 @@ import ( "fmt" "io" "os" + "time" + "github.com/containerd/containerd" "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/images" oci "github.com/containerd/containerd/images/oci" "github.com/containerd/containerd/log" "github.com/urfave/cli" @@ -53,23 +54,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb Usage: "image format. See DESCRIPTION.", }, cli.StringFlag{ - Name: "oci-name", - Value: "unknown/unknown", - Usage: "prefix added to either oci.v1 ref annotation or digest", + Name: "prefix,oci-name", + Value: "", + Usage: "prefix image name for added images", + }, + cli.BoolFlag{ + Name: "digests", + Usage: "whether to create digest images", }, - // TODO(AkihiroSuda): support commands.LabelFlag (for all children objects) }, commands.SnapshotterFlags...), Action: func(context *cli.Context) error { var ( - in = context.Args().First() - imageImporter images.Importer + in = context.Args().First() + opts []containerd.ImportOpt ) switch format := context.String("format"); format { case "oci.v1": - imageImporter = &oci.V1Importer{ - ImageName: context.String("oci-name"), + opts = append(opts, containerd.WithImporter(&oci.V1Importer{})) + + prefix := context.String("prefix") + if prefix == "" { + prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) + } + + opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix))) + if context.Bool("digests") { + opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix))) } default: return fmt.Errorf("unknown format %s", format) @@ -90,20 +102,24 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb return err } } - imgs, err := client.Import(ctx, imageImporter, r) + imgs, err := client.Import(ctx, r, opts...) + closeErr := r.Close() if err != nil { return err } - if err = r.Close(); err != nil { - return err + if closeErr != nil { + return closeErr } log.G(ctx).Debugf("unpacking %d images", len(imgs)) for _, img := range imgs { + // TODO: Allow configuration of the platform + image := containerd.NewImage(client, img) + // TODO: Show unpack status - fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest) - err = img.Unpack(ctx, context.String("snapshotter")) + fmt.Printf("unpacking %s (%s)...", img.Name, img.Target.Digest) + err = image.Unpack(ctx, context.String("snapshotter")) if err != nil { return err } diff --git a/images/importexport.go b/images/importexport.go index 04a55fd38..843adcadc 100644 --- a/images/importexport.go +++ b/images/importexport.go @@ -27,7 +27,7 @@ import ( // Importer is the interface for image importer. type Importer interface { // Import imports an image from a tar stream. - Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error) + Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) } // Exporter is the interface for image exporter. diff --git a/images/oci/importer.go b/images/oci/importer.go index 64efd3fc2..391d5b9f3 100644 --- a/images/oci/importer.go +++ b/images/oci/importer.go @@ -19,114 +19,89 @@ package oci import ( "archive/tar" + "bytes" "context" - "encoding/json" - "fmt" "io" "io/ioutil" "path" "strings" "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/log" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) // V1Importer implements OCI Image Spec v1. -type V1Importer struct { - // ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs). - // This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355 - ImageName string -} +type V1Importer struct{} var _ images.Importer = &V1Importer{} // Import implements Importer. -func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) { - if oi.ImageName == "" { - return nil, errors.New("ImageName not set") - } - tr := tar.NewReader(reader) - var imgrecs []images.Image - foundIndexJSON := false +func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { + var ( + desc ocispec.Descriptor + tr = tar.NewReader(reader) + ) for { hdr, err := tr.Next() if err == io.EOF { break } if err != nil { - return nil, err + return ocispec.Descriptor{}, err } if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") continue } hdrName := path.Clean(hdr.Name) if hdrName == "index.json" { - if foundIndexJSON { - return nil, errors.New("duplicated index.json") + if desc.Digest != "" { + return ocispec.Descriptor{}, errors.New("duplicated index.json") } - foundIndexJSON = true - imgrecs, err = onUntarIndexJSON(tr, oi.ImageName) + desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size) if err != nil { - return nil, err + return ocispec.Descriptor{}, err } - continue - } - if strings.HasPrefix(hdrName, "blobs/") { + } else if strings.HasPrefix(hdrName, "blobs/") { if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { - return nil, err + return ocispec.Descriptor{}, err } + } else if hdrName == ocispec.ImageLayoutFile { + // TODO Validate + } else { + log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored") } } - if !foundIndexJSON { - return nil, errors.New("no index.json found") + if desc.Digest == "" { + return ocispec.Descriptor{}, errors.New("no index.json found") } - for _, img := range imgrecs { - err := setGCRefContentLabels(ctx, store, img.Target) - if err != nil { - return imgrecs, err - } - } - // FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)? - return imgrecs, nil + + return desc, nil } -func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) { +func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) { b, err := ioutil.ReadAll(r) if err != nil { - return nil, err + return ocispec.Descriptor{}, err } - var idx ocispec.Index - if err := json.Unmarshal(b, &idx); err != nil { - return nil, err + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageIndex, + Digest: digest.FromBytes(b), + Size: size, } - var imgrecs []images.Image - for _, m := range idx.Manifests { - ref, err := normalizeImageRef(imageName, m) - if err != nil { - return nil, err - } - imgrecs = append(imgrecs, images.Image{ - Name: ref, - Target: m, - }) + if int64(len(b)) != size { + return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size) } - return imgrecs, nil -} -func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) { - digest := manifest.Digest - if digest == "" { - return "", errors.Errorf("manifest with empty digest: %v", manifest) + if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil { + return ocispec.Descriptor{}, err } - ociRef := manifest.Annotations[ocispec.AnnotationRefName] - if ociRef == "" { - return imageName + "@" + digest.String(), nil - } - return imageName + ":" + ociRef, nil + + return desc, err } func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error { @@ -140,65 +115,22 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name return errors.Errorf("unsupported algorithm: %s", algo) } dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) + return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) } -// GetChildrenDescriptors returns children blob descriptors for the following supported types: -// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest -// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex -func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - var manifest ocispec.Manifest - if err := json.NewDecoder(r).Decode(&manifest); err != nil { - return nil, err - } - return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil - case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: - var index ocispec.Index - if err := json.NewDecoder(r).Decode(&index); err != nil { - return nil, err - } - return index.Manifests, nil +// RefTranslator creates a reference using an OCI ref annotation, +// which is mentioned in the spec as only a tag compontent, +// concatenated with an image name +func RefTranslator(prefix string) func(string) string { + return func(ref string) string { + return prefix + ":" + ref } - return nil, nil } -func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error { - info, err := store.Info(ctx, desc.Digest) - if err != nil { - if errdefs.IsNotFound(err) { - // when the archive is created from multi-arch image, - // it may contain only blobs for a certain platform. - // So ErrNotFound (on manifest list) is expected here. - return nil - } - return err +// DigestTranslator creates a digest reference by adding the +// digest to an image name +func DigestTranslator(prefix string) func(digest.Digest) string { + return func(dgst digest.Digest) string { + return prefix + "@" + dgst.String() } - ra, err := store.ReaderAt(ctx, desc) - if err != nil { - return err - } - defer ra.Close() - r := content.NewReader(ra) - children, err := GetChildrenDescriptors(r, desc) - if err != nil { - return err - } - if info.Labels == nil { - info.Labels = map[string]string{} - } - for i, child := range children { - // Note: child blob is not guaranteed to be written to the content store. (multi-arch) - info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String() - } - if _, err := store.Update(ctx, info, "labels"); err != nil { - return err - } - for _, child := range children { - if err := setGCRefContentLabels(ctx, store, child); err != nil { - return err - } - } - return nil } diff --git a/images/oci/importer_test.go b/images/oci/importer_test.go deleted file mode 100644 index e8c950597..000000000 --- a/images/oci/importer_test.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package oci - -import ( - "testing" - - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "gotest.tools/assert" -) - -func TestNormalizeImageRef(t *testing.T) { - imageBaseName := "foo/bar" - for _, test := range []struct { - input ocispec.Descriptor - expect string - }{ - { - input: ocispec.Descriptor{ - Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"), - }, - expect: "foo/bar@sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77", - }, - { - input: ocispec.Descriptor{ - Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"), - Annotations: map[string]string{ - ocispec.AnnotationRefName: "latest", - }, - }, - expect: "foo/bar:latest", // no @digest for simplicity - }, - } { - normalized, err := normalizeImageRef(imageBaseName, test.input) - assert.NilError(t, err) - assert.Equal(t, test.expect, normalized) - } -} diff --git a/import.go b/import.go index 7a69f1d45..4e42a7429 100644 --- a/import.go +++ b/import.go @@ -18,35 +18,75 @@ package containerd import ( "context" + "encoding/json" "io" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/oci" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) type importOpts struct { + indexName string + imageRefT func(string) string + dgstRefT func(digest.Digest) string + importer images.Importer } // ImportOpt allows the caller to specify import specific options -type ImportOpt func(c *importOpts) error +type ImportOpt func(*importOpts) error -func resolveImportOpt(opts ...ImportOpt) (importOpts, error) { - var iopts importOpts - for _, o := range opts { - if err := o(&iopts); err != nil { - return iopts, err - } +// WithImageRefTranslator is used to translate the index reference +// to an image reference for the image store. +func WithImageRefTranslator(f func(string) string) ImportOpt { + return func(c *importOpts) error { + c.imageRefT = f + return nil + } +} + +// WithDigestRef is used to create digest images for each +// manifest in the index. +func WithDigestRef(f func(digest.Digest) string) ImportOpt { + return func(c *importOpts) error { + c.dgstRefT = f + return nil + } +} + +// WithIndexName creates a tag pointing to the imported index +func WithIndexName(name string) ImportOpt { + return func(c *importOpts) error { + c.indexName = name + return nil + } +} + +// WithImporter sets the importer to use for converting +// the read stream into an OCI Index. +func WithImporter(importer images.Importer) ImportOpt { + return func(c *importOpts) error { + c.importer = importer + return nil } - return iopts, nil } // Import imports an image from a Tar stream using reader. // Caller needs to specify importer. Future version may use oci.v1 as the default. // Note that unreferrenced blobs may be imported to the content store as well. -func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) { - _, err := resolveImportOpt(opts...) // unused now - if err != nil { - return nil, err +func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt) ([]images.Image, error) { + var iopts importOpts + for _, o := range opts { + if err := o(&iopts); err != nil { + return nil, err + } + } + + if iopts.importer == nil { + iopts.importer = &oci.V1Importer{} } ctx, done, err := c.WithLease(ctx) @@ -55,31 +95,86 @@ func (c *Client) Import(ctx context.Context, importer images.Importer, reader io } defer done(ctx) - imgrecs, err := importer.Import(ctx, c.ContentStore(), reader) + index, err := iopts.importer.Import(ctx, c.ContentStore(), reader) if err != nil { - // is.Update() is not called on error return nil, err } - is := c.ImageService() - var images []Image - for _, imgrec := range imgrecs { - if updated, err := is.Update(ctx, imgrec, "target"); err != nil { + var ( + imgs []images.Image + cs = c.ContentStore() + is = c.ImageService() + ) + + if iopts.indexName != "" { + imgs = append(imgs, images.Image{ + Name: iopts.indexName, + Target: index, + }) + } + + var handler images.HandlerFunc + handler = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + // Only save images at top level + if desc.Digest != index.Digest { + return images.Children(ctx, cs, desc) + } + + p, err := content.ReadBlob(ctx, cs, desc) + if err != nil { + return nil, err + } + + var idx ocispec.Index + if err := json.Unmarshal(p, &idx); err != nil { + return nil, err + } + + for _, m := range idx.Manifests { + if ref := m.Annotations[ocispec.AnnotationRefName]; ref != "" { + if iopts.imageRefT != nil { + ref = iopts.imageRefT(ref) + } + if ref != "" { + imgs = append(imgs, images.Image{ + Name: ref, + Target: m, + }) + } + } + if iopts.dgstRefT != nil { + ref := iopts.dgstRefT(m.Digest) + if ref != "" { + imgs = append(imgs, images.Image{ + Name: ref, + Target: m, + }) + } + } + } + + return idx.Manifests, nil + } + + handler = images.SetChildrenLabels(cs, handler) + if err := images.Walk(ctx, handler, index); err != nil { + return nil, err + } + + for i := range imgs { + img, err := is.Update(ctx, imgs[i], "target") + if err != nil { if !errdefs.IsNotFound(err) { return nil, err } - created, err := is.Create(ctx, imgrec) + img, err = is.Create(ctx, imgs[i]) if err != nil { return nil, err } - - imgrec = created - } else { - imgrec = updated } - - images = append(images, NewImage(c, imgrec)) + imgs[i] = img } - return images, nil + + return imgs, nil } diff --git a/import_test.go b/import_test.go index 62813ac0c..10205baca 100644 --- a/import_test.go +++ b/import_test.go @@ -49,13 +49,17 @@ func TestOCIExportAndImport(t *testing.T) { t.Fatal(err) } - imgrecs, err := client.Import(ctx, &oci.V1Importer{ImageName: "foo/bar:"}, exported) + opts := []ImportOpt{ + WithImporter(&oci.V1Importer{}), + WithImageRefTranslator(oci.RefTranslator("foo/bar")), + } + imgrecs, err := client.Import(ctx, exported, opts...) if err != nil { t.Fatal(err) } for _, imgrec := range imgrecs { - err = client.ImageService().Delete(ctx, imgrec.Name()) + err = client.ImageService().Delete(ctx, imgrec.Name) if err != nil { t.Fatal(err) }