From 63401970c7bad8498ceae7b4e3a6e22c449949d8 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 5 Oct 2017 08:52:10 +0000 Subject: [PATCH] importer: refactor - Use lease API (previoisly, GC was not supported) - Refactored interfaces for ease of future Docker v1 importer support For usage, please refer to `ctr images import --help`. Signed-off-by: Akihiro Suda --- client.go | 159 ++++++++--------------- cmd/ctr/commands/images/export.go | 14 ++- cmd/ctr/commands/images/import.go | 83 ++++++++---- export_test.go | 9 +- images/importexport.go | 21 ++++ export.go => images/oci/exporter.go | 35 +++--- images/oci/importer.go | 188 ++++++++++++++++++++++++++++ images/oci/importer_test.go | 37 ++++++ import.go | 120 ------------------ import_test.go | 19 +-- remotes/handlers.go | 1 + 11 files changed, 394 insertions(+), 292 deletions(-) create mode 100644 images/importexport.go rename export.go => images/oci/exporter.go (83%) create mode 100644 images/oci/importer.go create mode 100644 images/oci/importer_test.go delete mode 100644 import.go diff --git a/client.go b/client.go index f340e9509..58081199b 100644 --- a/client.go +++ b/client.go @@ -29,7 +29,6 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/plugin" - "github.com/containerd/containerd/reference" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/remotes/docker/schema1" @@ -494,95 +493,27 @@ func (c *Client) Version(ctx context.Context) (Version, error) { }, nil } -type imageFormat string - -const ( - ociImageFormat imageFormat = "oci" -) - type importOpts struct { - format imageFormat - refObject string - labels map[string]string } // ImportOpt allows the caller to specify import specific options type ImportOpt func(c *importOpts) error -// WithImportLabel sets a label to be associated with an imported image -func WithImportLabel(key, value string) ImportOpt { - return func(opts *importOpts) error { - if opts.labels == nil { - opts.labels = make(map[string]string) - } - - opts.labels[key] = value - return nil - } -} - -// WithImportLabels associates a set of labels to an imported image -func WithImportLabels(labels map[string]string) ImportOpt { - return func(opts *importOpts) error { - if opts.labels == nil { - opts.labels = make(map[string]string) - } - - for k, v := range labels { - opts.labels[k] = v - } - return nil - } -} - -// WithOCIImportFormat sets the import format for an OCI image format -func WithOCIImportFormat() ImportOpt { - return func(c *importOpts) error { - if c.format != "" { - return errors.New("format already set") - } - c.format = ociImageFormat - return nil - } -} - -// WithRefObject specifies the ref object to import. -// If refObject is empty, it is copied from the ref argument of Import(). -func WithRefObject(refObject string) ImportOpt { - return func(c *importOpts) error { - c.refObject = refObject - return nil - } -} - -func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) { +func resolveImportOpt(opts ...ImportOpt) (importOpts, error) { var iopts importOpts for _, o := range opts { if err := o(&iopts); err != nil { return iopts, err } } - // use OCI as the default format - if iopts.format == "" { - iopts.format = ociImageFormat - } - // if refObject is not explicitly specified, use the one specified in ref - if iopts.refObject == "" { - refSpec, err := reference.Parse(ref) - if err != nil { - return iopts, err - } - iopts.refObject = refSpec.Object - } return iopts, nil } // Import imports an image from a Tar stream using reader. -// OCI format is assumed by default. -// -// Note that unreferenced blobs are imported to the content store as well. -func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) { - iopts, err := resolveImportOpt(ref, opts...) +// Caller needs to specify importer. Future version may use oci.v1 as the default. +// Note that unreferrenced blobs may be imported to the content store as well. +func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) { + _, err := resolveImportOpt(opts...) // unused now if err != nil { return nil, err } @@ -593,58 +524,66 @@ func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts } defer done() - switch iopts.format { - case ociImageFormat: - return c.importFromOCITar(ctx, ref, reader, iopts) - default: - return nil, errors.Errorf("unsupported format: %s", iopts.format) + imgrecs, err := importer.Import(ctx, c.ContentStore(), reader) + if err != nil { + // is.Update() is not called on error + return nil, err } + + is := c.ImageService() + var images []Image + for _, imgrec := range imgrecs { + if updated, err := is.Update(ctx, imgrec, "target"); err != nil { + if !errdefs.IsNotFound(err) { + return nil, err + } + + created, err := is.Create(ctx, imgrec) + if err != nil { + return nil, err + } + + imgrec = created + } else { + imgrec = updated + } + + images = append(images, &image{ + client: c, + i: imgrec, + }) + } + return images, nil } type exportOpts struct { - format imageFormat } -// ExportOpt allows callers to set export options +// ExportOpt allows the caller to specify export-specific options type ExportOpt func(c *exportOpts) error -// WithOCIExportFormat sets the OCI image format as the export target -func WithOCIExportFormat() ExportOpt { - return func(c *exportOpts) error { - if c.format != "" { - return errors.New("format already set") +func resolveExportOpt(opts ...ExportOpt) (exportOpts, error) { + var eopts exportOpts + for _, o := range opts { + if err := o(&eopts); err != nil { + return eopts, err } - c.format = ociImageFormat - return nil } + return eopts, nil } -// TODO: add WithMediaTypeTranslation that transforms media types according to the format. -// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip -// -> application/vnd.oci.image.layer.v1.tar+gzip - // Export exports an image to a Tar stream. // OCI format is used by default. // It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. -func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) { - var eopts exportOpts - for _, o := range opts { - if err := o(&eopts); err != nil { - return nil, err - } - } - // use OCI as the default format - if eopts.format == "" { - eopts.format = ociImageFormat +// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream. +func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) { + _, err := resolveExportOpt(opts...) // unused now + if err != nil { + return nil, err } pr, pw := io.Pipe() - switch eopts.format { - case ociImageFormat: - go func() { - pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts)) - }() - default: - return nil, errors.Errorf("unsupported format: %s", eopts.format) - } + go func() { + pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw)) + }() return pr, nil } diff --git a/cmd/ctr/commands/images/export.go b/cmd/ctr/commands/images/export.go index 26821008b..3a972913b 100644 --- a/cmd/ctr/commands/images/export.go +++ b/cmd/ctr/commands/images/export.go @@ -5,6 +5,7 @@ import ( "os" "github.com/containerd/containerd/cmd/ctr/commands" + oci "github.com/containerd/containerd/images/oci" "github.com/containerd/containerd/reference" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -13,11 +14,14 @@ import ( ) var exportCommand = cli.Command{ - Name: "export", - Usage: "export an image", - ArgsUsage: "[flags] ", - Description: "export an image to a tar stream", + Name: "export", + Usage: "export an image", + ArgsUsage: "[flags] ", + Description: `Export an image to a tar stream. +Currently, only OCI format is supported. +`, Flags: []cli.Flag{ + // TODO(AkihiroSuda): make this map[string]string as in moby/moby#33355? cli.StringFlag{ Name: "oci-ref-name", Value: "", @@ -78,7 +82,7 @@ var exportCommand = cli.Command{ return nil } } - r, err := client.Export(ctx, desc) + r, err := client.Export(ctx, &oci.V1Exporter{}, desc) if err != nil { return err } diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index 74f204ea9..18af7dbdf 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -5,37 +5,66 @@ import ( "io" "os" - "github.com/containerd/containerd" "github.com/containerd/containerd/cmd/ctr/commands" + "github.com/containerd/containerd/images" + oci "github.com/containerd/containerd/images/oci" "github.com/containerd/containerd/log" "github.com/urfave/cli" ) var importCommand = cli.Command{ - Name: "import", - Usage: "import an image", - ArgsUsage: "[flags] ", - Description: "import an image from a tar stream", - Flags: []cli.Flag{ + Name: "import", + Usage: "import images", + ArgsUsage: "[flags] ", + Description: `Import images from a tar stream. +Implemented formats: +- oci.v1 (default) + + +For oci.v1 format, you need to specify --oci-name because an OCI archive contains image refs (tags) +but does not contain the base image name. + +e.g. + $ ctr images import --format oci.v1 --oci-name foo/bar foobar.tar + +If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadbeef", the command will create +"foo/bar:latest" and "foo/bar@sha256:deadbeef" images in the containerd store. +`, + Flags: append([]cli.Flag{ cli.StringFlag{ - Name: "ref-object", - Value: "", - Usage: "reference object e.g. tag@digest (default: use the object specified in ref)", + Name: "format", + Value: "oci.v1", + Usage: "image format. See DESCRIPTION.", }, - commands.LabelFlag, - }, + cli.StringFlag{ + Name: "oci-name", + Value: "unknown/unknown", + Usage: "prefix added to either oci.v1 ref annotation or digest", + }, + // TODO(AkihiroSuda): support commands.LabelFlag (for all children objects) + }, commands.SnapshotterFlags...), + Action: func(context *cli.Context) error { var ( - ref = context.Args().First() - in = context.Args().Get(1) - refObject = context.String("ref-object") - labels = commands.LabelArgs(context.StringSlice("label")) + in = context.Args().First() + imageImporter images.Importer ) + + switch format := context.String("format"); format { + case "oci.v1": + imageImporter = &oci.V1Importer{ + ImageName: context.String("oci-name"), + } + default: + return fmt.Errorf("unknown format %s", format) + } + client, ctx, cancel, err := commands.NewClient(context) if err != nil { return err } defer cancel() + var r io.ReadCloser if in == "-" { r = os.Stdin @@ -45,12 +74,7 @@ var importCommand = cli.Command{ return err } } - img, err := client.Import(ctx, - ref, - r, - containerd.WithRefObject(refObject), - containerd.WithImportLabels(labels), - ) + imgs, err := client.Import(ctx, imageImporter, r) if err != nil { return err } @@ -58,12 +82,17 @@ var importCommand = cli.Command{ return err } - log.G(ctx).WithField("image", ref).Debug("unpacking") + log.G(ctx).Debugf("unpacking %d images", len(imgs)) - // TODO: Show unpack status - fmt.Printf("unpacking %s...", img.Target().Digest) - err = img.Unpack(ctx, context.String("snapshotter")) - fmt.Println("done") - return err + for _, img := range imgs { + // TODO: Show unpack status + fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest) + err = img.Unpack(ctx, context.String("snapshotter")) + if err != nil { + return err + } + fmt.Println("done") + } + return nil }, } diff --git a/export_test.go b/export_test.go index dfe1d33f8..e6efc16a0 100644 --- a/export_test.go +++ b/export_test.go @@ -5,10 +5,12 @@ import ( "io" "runtime" "testing" + + "github.com/containerd/containerd/images/oci" ) -// TestExport exports testImage as a tar stream -func TestExport(t *testing.T) { +// TestOCIExport exports testImage as a tar stream +func TestOCIExport(t *testing.T) { // TODO: support windows if testing.Short() || runtime.GOOS == "windows" { t.Skip() @@ -26,8 +28,7 @@ func TestExport(t *testing.T) { if err != nil { t.Fatal(err) } - - exportedStream, err := client.Export(ctx, pulled.Target()) + exportedStream, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target()) if err != nil { t.Fatal(err) } diff --git a/images/importexport.go b/images/importexport.go new file mode 100644 index 000000000..f8cf742ba --- /dev/null +++ b/images/importexport.go @@ -0,0 +1,21 @@ +package images + +import ( + "context" + "io" + + "github.com/containerd/containerd/content" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Importer is the interface for image importer. +type Importer interface { + // Import imports an image from a tar stream. + Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error) +} + +// Exporter is the interface for image exporter. +type Exporter interface { + // Export exports an image to a tar stream. + Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error +} diff --git a/export.go b/images/oci/exporter.go similarity index 83% rename from export.go rename to images/oci/exporter.go index 76bebe3cd..9a559aa6e 100644 --- a/export.go +++ b/images/oci/exporter.go @@ -1,4 +1,4 @@ -package containerd +package oci import ( "archive/tar" @@ -15,7 +15,17 @@ import ( "github.com/pkg/errors" ) -func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error { +// V1Exporter implements OCI Image Spec v1. +// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. +// +// TODO(AkihiroSuda): add V1Exporter{TranslateMediaTypes: true} that transforms media types, +// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip +// -> application/vnd.oci.image.layer.v1.tar+gzip +type V1Exporter struct { +} + +// Export implements Exporter. +func (oe *V1Exporter) Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error { tw := tar.NewWriter(writer) defer tw.Close() @@ -24,16 +34,15 @@ func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, wr ociIndexRecord(desc), } - cs := c.ContentStore() algorithms := map[string]struct{}{} exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - records = append(records, blobRecord(cs, desc)) + records = append(records, blobRecord(store, desc)) algorithms[desc.Digest.Algorithm().String()] = struct{}{} return nil, nil } handlers := images.Handlers( - images.ChildrenHandler(cs, platforms.Default()), + images.ChildrenHandler(store, platforms.Default()), images.HandlerFunc(exportHandler), ) @@ -155,7 +164,9 @@ func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord { } func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error { - sort.Sort(tarRecordsByName(records)) + sort.Slice(records, func(i, j int) bool { + return records[i].Header.Name < records[j].Header.Name + }) for _, record := range records { if err := tw.WriteHeader(record.Header); err != nil { @@ -175,15 +186,3 @@ func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error { } return nil } - -type tarRecordsByName []tarRecord - -func (t tarRecordsByName) Len() int { - return len(t) -} -func (t tarRecordsByName) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} -func (t tarRecordsByName) Less(i, j int) bool { - return t[i].Header.Name < t[j].Header.Name -} diff --git a/images/oci/importer.go b/images/oci/importer.go new file mode 100644 index 000000000..f024fafe8 --- /dev/null +++ b/images/oci/importer.go @@ -0,0 +1,188 @@ +// Package oci provides the importer and the exporter for OCI Image Spec. +package oci + +import ( + "archive/tar" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "path" + "strings" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// V1Importer implements OCI Image Spec v1. +type V1Importer struct { + // ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs). + // This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355 + ImageName string +} + +var _ images.Importer = &V1Importer{} + +// Import implements Importer. +func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) { + if oi.ImageName == "" { + return nil, errors.New("ImageName not set") + } + tr := tar.NewReader(reader) + var imgrecs []images.Image + foundIndexJSON := false + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + continue + } + hdrName := path.Clean(hdr.Name) + if hdrName == "index.json" { + if foundIndexJSON { + return nil, errors.New("duplicated index.json") + } + foundIndexJSON = true + imgrecs, err = onUntarIndexJSON(tr, oi.ImageName) + if err != nil { + return nil, err + } + continue + } + if strings.HasPrefix(hdrName, "blobs/") { + if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { + return nil, err + } + } + } + if !foundIndexJSON { + return nil, errors.New("no index.json found") + } + for _, img := range imgrecs { + err := setGCRefContentLabels(ctx, store, img.Target) + if err != nil { + return imgrecs, err + } + } + // FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)? + return imgrecs, nil +} + +func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) { + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + var idx ocispec.Index + if err := json.Unmarshal(b, &idx); err != nil { + return nil, err + } + var imgrecs []images.Image + for _, m := range idx.Manifests { + ref, err := normalizeImageRef(imageName, m) + if err != nil { + return nil, err + } + imgrecs = append(imgrecs, images.Image{ + Name: ref, + Target: m, + }) + } + return imgrecs, nil +} + +func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) { + digest := manifest.Digest + if digest == "" { + return "", errors.Errorf("manifest with empty digest: %v", manifest) + } + ociRef := manifest.Annotations[ocispec.AnnotationRefName] + if ociRef == "" { + return imageName + "@" + digest.String(), nil + } + return imageName + ":" + ociRef, nil +} + +func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error { + // name is like "blobs/sha256/deadbeef" + split := strings.Split(name, "/") + if len(split) != 3 { + return errors.Errorf("unexpected name: %q", name) + } + algo := digest.Algorithm(split[1]) + if !algo.Available() { + return errors.Errorf("unsupported algorithm: %s", algo) + } + dgst := digest.NewDigestFromHex(algo.String(), split[2]) + return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst) +} + +// GetChildrenDescriptors returns children blob descriptors for the following supported types: +// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest +// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex +func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + switch desc.MediaType { + case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + var manifest ocispec.Manifest + if err := json.NewDecoder(r).Decode(&manifest); err != nil { + return nil, err + } + return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil + case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + var index ocispec.Index + if err := json.NewDecoder(r).Decode(&index); err != nil { + return nil, err + } + return index.Manifests, nil + } + return nil, nil +} + +func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error { + info, err := store.Info(ctx, desc.Digest) + if err != nil { + if errdefs.IsNotFound(err) { + // when the archive is created from multi-arch image, + // it may contain only blobs for a certain platform. + // So ErrNotFound (on manifest list) is expected here. + return nil + } + return err + } + ra, err := store.ReaderAt(ctx, desc.Digest) + if err != nil { + return err + } + defer ra.Close() + r := content.NewReader(ra) + children, err := GetChildrenDescriptors(r, desc) + if err != nil { + return err + } + if info.Labels == nil { + info.Labels = map[string]string{} + } + for i, child := range children { + // Note: child blob is not guaranteed to be written to the content store. (multi-arch) + info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String() + } + if _, err := store.Update(ctx, info, "labels"); err != nil { + return err + } + for _, child := range children { + if err := setGCRefContentLabels(ctx, store, child); err != nil { + return err + } + } + return nil +} diff --git a/images/oci/importer_test.go b/images/oci/importer_test.go new file mode 100644 index 000000000..ba0e8d6c1 --- /dev/null +++ b/images/oci/importer_test.go @@ -0,0 +1,37 @@ +package oci + +import ( + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" +) + +func TestNormalizeImageRef(t *testing.T) { + imageBaseName := "foo/bar" + for _, test := range []struct { + input ocispec.Descriptor + expect string + }{ + { + input: ocispec.Descriptor{ + Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"), + }, + expect: "foo/bar@sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77", + }, + { + input: ocispec.Descriptor{ + Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"), + Annotations: map[string]string{ + ocispec.AnnotationRefName: "latest", + }, + }, + expect: "foo/bar:latest", // no @digest for simplicity + }, + } { + normalized, err := normalizeImageRef(imageBaseName, test.input) + assert.NoError(t, err) + assert.Equal(t, test.expect, normalized) + } +} diff --git a/import.go b/import.go deleted file mode 100644 index 9f8f9af7d..000000000 --- a/import.go +++ /dev/null @@ -1,120 +0,0 @@ -package containerd - -import ( - "archive/tar" - "context" - "encoding/json" - "io" - "io/ioutil" - "strings" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/reference" - digest "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" -) - -func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) { - tag, dgst := reference.SplitObject(refObject) - if tag == "" && dgst == "" { - return nil, errors.Errorf("unexpected object: %q", refObject) - } - for _, m := range idx.Manifests { - if m.Digest == dgst { - return &m, nil - } - annot, ok := m.Annotations[ocispec.AnnotationRefName] - if ok && annot == tag && tag != "" { - return &m, nil - } - } - return nil, errors.Errorf("not found: %q", refObject) -} - -func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) { - tr := tar.NewReader(reader) - store := c.ContentStore() - var desc *ocispec.Descriptor - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { - continue - } - if hdr.Name == "index.json" { - desc, err = onUntarIndexJSON(tr, iopts.refObject) - if err != nil { - return nil, err - } - continue - } - if strings.HasPrefix(hdr.Name, "blobs/") { - if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil { - return nil, err - } - } - } - if desc == nil { - return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject) - } - imgrec := images.Image{ - Name: ref, - Target: *desc, - Labels: iopts.labels, - } - is := c.ImageService() - if updated, err := is.Update(ctx, imgrec, "target"); err != nil { - if !errdefs.IsNotFound(err) { - return nil, err - } - - created, err := is.Create(ctx, imgrec) - if err != nil { - return nil, err - } - - imgrec = created - } else { - imgrec = updated - } - - img := &image{ - client: c, - i: imgrec, - } - return img, nil -} - -func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) { - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - var idx ocispec.Index - if err := json.Unmarshal(b, &idx); err != nil { - return nil, err - } - return resolveOCIIndex(idx, refObject) -} - -func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error { - // name is like "blobs/sha256/deadbeef" - split := strings.Split(name, "/") - if len(split) != 3 { - return errors.Errorf("unexpected name: %q", name) - } - algo := digest.Algorithm(split[1]) - if !algo.Available() { - return errors.Errorf("unsupported algorithm: %s", algo) - } - dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst) -} diff --git a/import_test.go b/import_test.go index 78640afda..56c5f9734 100644 --- a/import_test.go +++ b/import_test.go @@ -3,11 +3,13 @@ package containerd import ( "runtime" "testing" + + "github.com/containerd/containerd/images/oci" ) -// TestExportAndImport exports testImage as a tar stream, +// TestOCIExportAndImport exports testImage as a tar stream, // and import the tar stream as a new image. -func TestExportAndImport(t *testing.T) { +func TestOCIExportAndImport(t *testing.T) { // TODO: support windows if testing.Short() || runtime.GOOS == "windows" { t.Skip() @@ -26,19 +28,20 @@ func TestExportAndImport(t *testing.T) { t.Fatal(err) } - exported, err := client.Export(ctx, pulled.Target()) + exported, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target()) if err != nil { t.Fatal(err) } - importRef := "test/export-and-import:tmp" - _, err = client.Import(ctx, importRef, exported, WithRefObject("@"+pulled.Target().Digest.String())) + imgrecs, err := client.Import(ctx, &oci.V1Importer{ImageName: "foo/bar:"}, exported) if err != nil { t.Fatal(err) } - err = client.ImageService().Delete(ctx, importRef) - if err != nil { - t.Fatal(err) + for _, imgrec := range imgrecs { + err = client.ImageService().Delete(ctx, imgrec.Name()) + if err != nil { + t.Fatal(err) + } } } diff --git a/remotes/handlers.go b/remotes/handlers.go index e583391d8..ad4cd9f31 100644 --- a/remotes/handlers.go +++ b/remotes/handlers.go @@ -114,6 +114,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc func commitOpts(desc ocispec.Descriptor, r io.Reader) (io.Reader, []content.Opt) { var childrenF func(r io.Reader) ([]ocispec.Descriptor, error) + // TODO(AkihiroSuda): use images/oci.GetChildrenDescriptors? switch desc.MediaType { case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: childrenF = func(r io.Reader) ([]ocispec.Descriptor, error) {