From 4754d2aeee1d1b978baf84b645979e92f59bdaae Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 1 Apr 2019 15:20:46 -0700 Subject: [PATCH] Update image export to support Docker format Add manifest.json file which is used by Docker to import images. Signed-off-by: Derek McGowan --- cmd/ctr/commands/images/export.go | 123 +++----- export.go | 26 +- export_test.go | 15 +- images/annotations.go | 28 ++ images/archive/exporter.go | 490 ++++++++++++++++++++++++++++++ images/archive/reference.go | 10 + images/oci/exporter.go | 241 --------------- import.go | 49 ++- import_test.go | 15 +- 9 files changed, 643 insertions(+), 354 deletions(-) create mode 100644 images/annotations.go create mode 100644 images/archive/exporter.go delete mode 100644 images/oci/exporter.go diff --git a/cmd/ctr/commands/images/export.go b/cmd/ctr/commands/images/export.go index 629f49330..fc06a53ed 100644 --- a/cmd/ctr/commands/images/export.go +++ b/cmd/ctr/commands/images/export.go @@ -21,9 +21,8 @@ import ( "os" "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/images/oci" - "github.com/containerd/containerd/reference" - digest "github.com/opencontainers/go-digest" + "github.com/containerd/containerd/images/archive" + "github.com/containerd/containerd/platforms" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/urfave/cli" @@ -31,26 +30,24 @@ import ( var exportCommand = cli.Command{ Name: "export", - Usage: "export an image", - ArgsUsage: "[flags] ", - Description: `Export an image to a tar stream. -Currently, only OCI format is supported. + Usage: "export images", + ArgsUsage: "[flags] ...", + Description: `Export images to an OCI tar archive. + +Tar output is formatted as an OCI archive, a Docker manifest is provided for the platform. +Use '--skip-manifest-json' to avoid including the Docker manifest.json file. +Use '--platform' to define the output platform. +When '--all-platforms' is given all images in a manifest list must be available. `, Flags: []cli.Flag{ - // TODO(AkihiroSuda): make this map[string]string as in moby/moby#33355? - cli.StringFlag{ - Name: "oci-ref-name", - Value: "", - Usage: "override org.opencontainers.image.ref.name annotation", + cli.BoolFlag{ + Name: "skip-manifest-json", + Usage: "do not add Docker compatible manifest.json to archive", }, - cli.StringFlag{ - Name: "manifest", - Usage: "digest of manifest", - }, - cli.StringFlag{ - Name: "manifest-type", - Usage: "media type of manifest digest", - Value: ocispec.MediaTypeImageManifest, + cli.StringSliceFlag{ + Name: "platform", + Usage: "Pull content from a specific platform", + Value: &cli.StringSlice{}, }, cli.BoolFlag{ Name: "all-platforms", @@ -59,43 +56,47 @@ Currently, only OCI format is supported. }, Action: func(context *cli.Context) error { var ( - out = context.Args().First() - local = context.Args().Get(1) - desc ocispec.Descriptor + out = context.Args().First() + images = context.Args().Tail() + exportOpts = []archive.ExportOpt{} ) - if out == "" || local == "" { + if out == "" || len(images) == 0 { return errors.New("please provide both an output filename and an image reference to export") } + + if pss := context.StringSlice("platform"); len(pss) > 0 { + var all []ocispec.Platform + for _, ps := range pss { + p, err := platforms.Parse(ps) + if err != nil { + return errors.Wrapf(err, "invalid platform %q", ps) + } + all = append(all, p) + } + exportOpts = append(exportOpts, archive.WithPlatform(platforms.Ordered(all...))) + } else { + exportOpts = append(exportOpts, archive.WithPlatform(platforms.Default())) + } + + if context.Bool("all-platforms") { + exportOpts = append(exportOpts, archive.WithAllPlatforms()) + } + + if context.Bool("skip-manifest-json") { + exportOpts = append(exportOpts, archive.WithSkipDockerManifest()) + } + client, ctx, cancel, err := commands.NewClient(context) if err != nil { return err } defer cancel() - if manifest := context.String("manifest"); manifest != "" { - desc.Digest, err = digest.Parse(manifest) - if err != nil { - return errors.Wrap(err, "invalid manifest digest") - } - desc.MediaType = context.String("manifest-type") - } else { - img, err := client.ImageService().Get(ctx, local) - if err != nil { - return errors.Wrap(err, "unable to resolve image to manifest") - } - desc = img.Target + + is := client.ImageService() + for _, img := range images { + exportOpts = append(exportOpts, archive.WithImage(is, img)) } - if desc.Annotations == nil { - desc.Annotations = make(map[string]string) - } - if s, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok || s == "" { - if ociRefName := determineOCIRefName(local); ociRefName != "" { - desc.Annotations[ocispec.AnnotationRefName] = ociRefName - } - if ociRefName := context.String("oci-ref-name"); ociRefName != "" { - desc.Annotations[ocispec.AnnotationRefName] = ociRefName - } - } var w io.WriteCloser if out == "-" { w = os.Stdout @@ -105,32 +106,8 @@ Currently, only OCI format is supported. return nil } } + defer w.Close() - var ( - exportOpts []oci.V1ExporterOpt - ) - - exportOpts = append(exportOpts, oci.WithAllPlatforms(context.Bool("all-platforms"))) - - r, err := client.Export(ctx, desc, exportOpts...) - if err != nil { - return err - } - if _, err := io.Copy(w, r); err != nil { - return err - } - if err := w.Close(); err != nil { - return err - } - return r.Close() + return client.Export(ctx, w, exportOpts...) }, } - -func determineOCIRefName(local string) string { - refspec, err := reference.Parse(local) - if err != nil { - return "" - } - tag, _ := reference.SplitObject(refspec.Object) - return tag -} diff --git a/export.go b/export.go index f5552231e..81f199226 100644 --- a/export.go +++ b/export.go @@ -20,26 +20,12 @@ import ( "context" "io" - "github.com/containerd/containerd/images/oci" - - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" + "github.com/containerd/containerd/images/archive" ) -// Export exports an image to a Tar stream. -// OCI format is used by default. -// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. -// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream. -func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...oci.V1ExporterOpt) (io.ReadCloser, error) { - - exporter, err := oci.ResolveV1ExportOpt(opts...) - if err != nil { - return nil, err - } - - pr, pw := io.Pipe() - go func() { - pw.CloseWithError(errors.Wrap(exporter.Export(ctx, c.ContentStore(), desc, pw), "export failed")) - }() - return pr, nil +// Export exports images to a Tar stream. +// The tar archive is in OCI format with a Docker compatible manifest +// when a single target platform is given. +func (c *Client) Export(ctx context.Context, w io.Writer, opts ...archive.ExportOpt) error { + return archive.Export(ctx, c.ContentStore(), w, opts...) } diff --git a/export_test.go b/export_test.go index 9e200dc79..6b6dc98f8 100644 --- a/export_test.go +++ b/export_test.go @@ -18,13 +18,17 @@ package containerd import ( "archive/tar" + "bytes" "io" "runtime" "testing" + + "github.com/containerd/containerd/images/archive" + "github.com/containerd/containerd/platforms" ) -// TestOCIExport exports testImage as a tar stream -func TestOCIExport(t *testing.T) { +// TestExport exports testImage as a tar stream +func TestExport(t *testing.T) { // TODO: support windows if testing.Short() || runtime.GOOS == "windows" { t.Skip() @@ -38,15 +42,16 @@ func TestOCIExport(t *testing.T) { } defer client.Close() - pulled, err := client.Fetch(ctx, testImage) + _, err = client.Fetch(ctx, testImage) if err != nil { t.Fatal(err) } - exportedStream, err := client.Export(ctx, pulled.Target) + wb := bytes.NewBuffer(nil) + err = client.Export(ctx, wb, archive.WithPlatform(platforms.Default()), archive.WithImage(client.ImageService(), testImage)) if err != nil { t.Fatal(err) } - assertOCITar(t, exportedStream) + assertOCITar(t, bytes.NewReader(wb.Bytes())) } func assertOCITar(t *testing.T, r io.Reader) { diff --git a/images/annotations.go b/images/annotations.go new file mode 100644 index 000000000..bf9df7984 --- /dev/null +++ b/images/annotations.go @@ -0,0 +1,28 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package images + +const ( + // AnnotationImageName is an annotation on a Descriptor in an index.json + // containing the `Name` value as used by an `Image` struct + AnnotationImageName = "io.containerd.image.name" + + // AnnotationImageNamePrefix is used the same way as AnnotationImageName + // but may be used to refer to additional names in the annotation map + // using user-defined suffixes (i.e. "extra.1") + AnnotationImageNamePrefix = AnnotationImageName + "." +) diff --git a/images/archive/exporter.go b/images/archive/exporter.go new file mode 100644 index 000000000..b7a9d495f --- /dev/null +++ b/images/archive/exporter.go @@ -0,0 +1,490 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package archive + +import ( + "archive/tar" + "context" + "encoding/json" + "fmt" + "io" + "path" + "sort" + "strings" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +type exportOptions struct { + manifests []ocispec.Descriptor + platform platforms.MatchComparer + allPlatforms bool + skipDockerManifest bool +} + +// ExportOpt defines options for configuring exported descriptors +type ExportOpt func(context.Context, *exportOptions) error + +// WithPlatform defines the platform to require manifest lists have +// not exporting all platforms. +// Additionally, platform is used to resolve image configs for +// Docker v1.1, v1.2 format compatibility. +func WithPlatform(p platforms.MatchComparer) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + o.platform = p + return nil + } +} + +// WithAllPlatforms exports all manifests from a manifest list. +// Missing content will fail the export. +func WithAllPlatforms() ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + o.allPlatforms = true + return nil + } +} + +// WithSkipDockerManifest skips creation of the Docker compatible +// manifest.json file. +func WithSkipDockerManifest() ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + o.skipDockerManifest = true + return nil + } +} + +// WithImage adds the provided images to the exported archive. +func WithImage(is images.Store, name string) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + img, err := is.Get(ctx, name) + if err != nil { + return err + } + + var i int + o.manifests, i = appendDescriptor(o.manifests, img.Target) + o.manifests[i].Annotations = addNameAnnotation(name, o.manifests[i].Annotations) + + return nil + } +} + +// WithManifest adds a manifest to the exported archive. +// It is up to caller to put name annotation to on the manifest +// descriptor if needed. +func WithManifest(manifest ocispec.Descriptor) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + var i int + o.manifests, i = appendDescriptor(o.manifests, manifest) + o.manifests[i].Annotations = manifest.Annotations + return nil + } +} + +// WithNamedManifest adds a manifest to the exported archive +// with the provided names. +func WithNamedManifest(manifest ocispec.Descriptor, names ...string) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + var i int + o.manifests, i = appendDescriptor(o.manifests, manifest) + for _, name := range names { + o.manifests[i].Annotations = addNameAnnotation(name, o.manifests[i].Annotations) + } + + return nil + } +} + +func appendDescriptor(descs []ocispec.Descriptor, desc ocispec.Descriptor) ([]ocispec.Descriptor, int) { + i := 0 + for i < len(descs) { + if descs[i].Digest == desc.Digest { + return descs, i + } + i++ + } + return append(descs, desc), i +} + +func addNameAnnotation(name string, annotations map[string]string) map[string]string { + if annotations == nil { + annotations = map[string]string{} + } + + i := 0 + for { + key := images.AnnotationImageName + if i > 0 { + key = fmt.Sprintf("%sextra.%d", images.AnnotationImageNamePrefix, i) + } + i++ + + if val, ok := annotations[key]; ok { + if val != name { + continue + } + } else { + annotations[key] = name + } + break + } + return annotations +} + +// Export implements Exporter. +func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error { + var eo exportOptions + for _, opt := range opts { + if err := opt(ctx, &eo); err != nil { + return err + } + } + + records := []tarRecord{ + ociLayoutFile(""), + ociIndexRecord(eo.manifests), + } + + algorithms := map[string]struct{}{} + manifestTags := map[string]ocispec.Descriptor{} + for _, desc := range eo.manifests { + switch desc.MediaType { + case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + r, err := getRecords(ctx, store, desc, algorithms) + if err != nil { + return err + } + records = append(records, r...) + + for _, name := range imageNames(desc.Annotations) { + manifestTags[name] = desc + } + case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + records = append(records, blobRecord(store, desc)) + + p, err := content.ReadBlob(ctx, store, desc) + if err != nil { + return err + } + + var index ocispec.Index + if err := json.Unmarshal(p, &index); err != nil { + return err + } + + names := imageNames(desc.Annotations) + var manifests []ocispec.Descriptor + for _, m := range index.Manifests { + if eo.platform != nil { + if m.Platform == nil || eo.platform.Match(*m.Platform) { + manifests = append(manifests, m) + } else if !eo.allPlatforms { + continue + } + } + + r, err := getRecords(ctx, store, m, algorithms) + if err != nil { + return err + } + + records = append(records, r...) + } + + if len(names) > 0 && !eo.skipDockerManifest { + if len(manifests) >= 1 { + if len(manifests) > 1 { + sort.SliceStable(manifests, func(i, j int) bool { + if manifests[i].Platform == nil { + return false + } + if manifests[j].Platform == nil { + return true + } + return eo.platform.Less(*manifests[i].Platform, *manifests[j].Platform) + }) + } + for _, name := range names { + manifestTags[name] = manifests[0] + } + } else if eo.platform != nil { + return errors.Wrap(errdefs.ErrNotFound, "no manifest found for platform") + } + } + default: + return errors.Wrap(errdefs.ErrInvalidArgument, "only manifests may be exported") + } + } + + if len(manifestTags) > 0 { + tr, err := manifestsRecord(ctx, store, manifestTags) + if err != nil { + return errors.Wrap(err, "unable to create manifests file") + } + + records = append(records, tr) + } + + if len(algorithms) > 0 { + records = append(records, directoryRecord("blobs/", 0755)) + for alg := range algorithms { + records = append(records, directoryRecord("blobs/"+alg+"/", 0755)) + } + } + + tw := tar.NewWriter(writer) + defer tw.Close() + return writeTar(ctx, tw, records) +} + +func imageNames(annotations map[string]string) []string { + var names []string + for k, v := range annotations { + if k == images.AnnotationImageName || strings.HasPrefix(k, images.AnnotationImageName) { + names = append(names, v) + } + } + return names +} + +func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descriptor, algorithms map[string]struct{}) ([]tarRecord, error) { + var records []tarRecord + exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + records = append(records, blobRecord(store, desc)) + algorithms[desc.Digest.Algorithm().String()] = struct{}{} + return nil, nil + } + + childrenHandler := images.ChildrenHandler(store) + + handlers := images.Handlers( + childrenHandler, + images.HandlerFunc(exportHandler), + ) + + // Walk sequentially since the number of fetchs is likely one and doing in + // parallel requires locking the export handler + if err := images.Walk(ctx, handlers, desc); err != nil { + return nil, err + } + + return records, nil +} + +type tarRecord struct { + Header *tar.Header + CopyTo func(context.Context, io.Writer) (int64, error) +} + +func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord { + path := path.Join("blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded()) + return tarRecord{ + Header: &tar.Header{ + Name: path, + Mode: 0444, + Size: desc.Size, + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + r, err := cs.ReaderAt(ctx, desc) + if err != nil { + return 0, errors.Wrap(err, "failed to get reader") + } + defer r.Close() + + // Verify digest + dgstr := desc.Digest.Algorithm().Digester() + + n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) + if err != nil { + return 0, errors.Wrap(err, "failed to copy to tar") + } + if dgstr.Digest() != desc.Digest { + return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest()) + } + return n, nil + }, + } +} + +func directoryRecord(name string, mode int64) tarRecord { + return tarRecord{ + Header: &tar.Header{ + Name: name, + Mode: mode, + Typeflag: tar.TypeDir, + }, + } +} + +func ociLayoutFile(version string) tarRecord { + if version == "" { + version = ocispec.ImageLayoutVersion + } + layout := ocispec.ImageLayout{ + Version: version, + } + + b, err := json.Marshal(layout) + if err != nil { + panic(err) + } + + return tarRecord{ + Header: &tar.Header{ + Name: ocispec.ImageLayoutFile, + Mode: 0444, + Size: int64(len(b)), + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + n, err := w.Write(b) + return int64(n), err + }, + } + +} + +func ociIndexRecord(manifests []ocispec.Descriptor) tarRecord { + index := ocispec.Index{ + Versioned: ocispecs.Versioned{ + SchemaVersion: 2, + }, + Manifests: manifests, + } + + b, err := json.Marshal(index) + if err != nil { + panic(err) + } + + return tarRecord{ + Header: &tar.Header{ + Name: "index.json", + Mode: 0644, + Size: int64(len(b)), + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + n, err := w.Write(b) + return int64(n), err + }, + } +} + +func manifestsRecord(ctx context.Context, store content.Provider, manifests map[string]ocispec.Descriptor) (tarRecord, error) { + type mfst struct { + Config string + RepoTags []string + Layers []string + } + + images := map[digest.Digest]mfst{} + for name, m := range manifests { + p, err := content.ReadBlob(ctx, store, m) + if err != nil { + return tarRecord{}, err + } + + var manifest ocispec.Manifest + if err := json.Unmarshal(p, &manifest); err != nil { + return tarRecord{}, err + } + if err := manifest.Config.Digest.Validate(); err != nil { + return tarRecord{}, errors.Wrapf(err, "invalid manifest %q", m.Digest) + } + + nname, err := familiarizeReference(name) + if err != nil { + return tarRecord{}, err + } + + dgst := manifest.Config.Digest + mf, ok := images[dgst] + if !ok { + mf.Config = path.Join("blobs", dgst.Algorithm().String(), dgst.Encoded()) + for _, l := range manifest.Layers { + path := path.Join("blobs", l.Digest.Algorithm().String(), l.Digest.Encoded()) + mf.Layers = append(mf.Layers, path) + } + } + + mf.RepoTags = append(mf.RepoTags, nname) + + images[dgst] = mf + } + + var mfsts []mfst + for _, mf := range images { + mfsts = append(mfsts, mf) + } + + b, err := json.Marshal(mfsts) + if err != nil { + return tarRecord{}, err + } + + return tarRecord{ + Header: &tar.Header{ + Name: "manifest.json", + Mode: 0644, + Size: int64(len(b)), + Typeflag: tar.TypeReg, + }, + CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { + n, err := w.Write(b) + return int64(n), err + }, + }, nil +} + +func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error { + sort.Slice(records, func(i, j int) bool { + return records[i].Header.Name < records[j].Header.Name + }) + + var last string + for _, record := range records { + if record.Header.Name == last { + continue + } + last = record.Header.Name + if err := tw.WriteHeader(record.Header); err != nil { + return err + } + if record.CopyTo != nil { + n, err := record.CopyTo(ctx, tw) + if err != nil { + return err + } + if n != record.Header.Size { + return errors.Errorf("unexpected copy size for %s", record.Header.Name) + } + } else if record.Header.Size > 0 { + return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name) + } + } + return nil +} diff --git a/images/archive/reference.go b/images/archive/reference.go index 2e80a968a..5ac3e11b0 100644 --- a/images/archive/reference.go +++ b/images/archive/reference.go @@ -77,6 +77,16 @@ func normalizeReference(ref string) (string, error) { return normalized.String(), nil } +func familiarizeReference(ref string) (string, error) { + named, err := reference.ParseNormalizedNamed(ref) + if err != nil { + return "", errors.Wrapf(err, "failed to parse %q", ref) + } + named = reference.TagNameOnly(named) + + return reference.FamiliarString(named), nil +} + // DigestTranslator creates a digest reference by adding the // digest to an image name func DigestTranslator(prefix string) func(digest.Digest) string { diff --git a/images/oci/exporter.go b/images/oci/exporter.go deleted file mode 100644 index 8bb535489..000000000 --- a/images/oci/exporter.go +++ /dev/null @@ -1,241 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package oci - -import ( - "archive/tar" - "context" - "encoding/json" - "io" - "sort" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/platforms" - ocispecs "github.com/opencontainers/image-spec/specs-go" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" -) - -// V1Exporter implements OCI Image Spec v1. -// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. -// -// TODO(AkihiroSuda): add V1Exporter{TranslateMediaTypes: true} that transforms media types, -// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip -// -> application/vnd.oci.image.layer.v1.tar+gzip -type V1Exporter struct { - AllPlatforms bool -} - -// V1ExporterOpt allows the caller to set additional options to a new V1Exporter -type V1ExporterOpt func(c *V1Exporter) error - -// DefaultV1Exporter return a default V1Exporter pointer -func DefaultV1Exporter() *V1Exporter { - return &V1Exporter{ - AllPlatforms: false, - } -} - -// ResolveV1ExportOpt return a new V1Exporter with V1ExporterOpt -func ResolveV1ExportOpt(opts ...V1ExporterOpt) (*V1Exporter, error) { - exporter := DefaultV1Exporter() - for _, o := range opts { - if err := o(exporter); err != nil { - return exporter, err - } - } - return exporter, nil -} - -// WithAllPlatforms set V1Exporter`s AllPlatforms option -func WithAllPlatforms(allPlatforms bool) V1ExporterOpt { - return func(c *V1Exporter) error { - c.AllPlatforms = allPlatforms - return nil - } -} - -// Export implements Exporter. -func (oe *V1Exporter) Export(ctx context.Context, store content.Provider, desc ocispec.Descriptor, writer io.Writer) error { - tw := tar.NewWriter(writer) - defer tw.Close() - - records := []tarRecord{ - ociLayoutFile(""), - ociIndexRecord(desc), - } - - algorithms := map[string]struct{}{} - exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - records = append(records, blobRecord(store, desc)) - algorithms[desc.Digest.Algorithm().String()] = struct{}{} - return nil, nil - } - - childrenHandler := images.ChildrenHandler(store) - - if !oe.AllPlatforms { - // get local default platform to fetch image manifest - childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Any(platforms.DefaultSpec())) - } - - handlers := images.Handlers( - childrenHandler, - images.HandlerFunc(exportHandler), - ) - - // Walk sequentially since the number of fetchs is likely one and doing in - // parallel requires locking the export handler - if err := images.Walk(ctx, handlers, desc); err != nil { - return err - } - - if len(algorithms) > 0 { - records = append(records, directoryRecord("blobs/", 0755)) - for alg := range algorithms { - records = append(records, directoryRecord("blobs/"+alg+"/", 0755)) - } - } - - return writeTar(ctx, tw, records) -} - -type tarRecord struct { - Header *tar.Header - CopyTo func(context.Context, io.Writer) (int64, error) -} - -func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord { - path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex() - return tarRecord{ - Header: &tar.Header{ - Name: path, - Mode: 0444, - Size: desc.Size, - Typeflag: tar.TypeReg, - }, - CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { - r, err := cs.ReaderAt(ctx, desc) - if err != nil { - return 0, errors.Wrap(err, "failed to get reader") - } - defer r.Close() - - // Verify digest - dgstr := desc.Digest.Algorithm().Digester() - - n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) - if err != nil { - return 0, errors.Wrap(err, "failed to copy to tar") - } - if dgstr.Digest() != desc.Digest { - return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest()) - } - return n, nil - }, - } -} - -func directoryRecord(name string, mode int64) tarRecord { - return tarRecord{ - Header: &tar.Header{ - Name: name, - Mode: mode, - Typeflag: tar.TypeDir, - }, - } -} - -func ociLayoutFile(version string) tarRecord { - if version == "" { - version = ocispec.ImageLayoutVersion - } - layout := ocispec.ImageLayout{ - Version: version, - } - - b, err := json.Marshal(layout) - if err != nil { - panic(err) - } - - return tarRecord{ - Header: &tar.Header{ - Name: ocispec.ImageLayoutFile, - Mode: 0444, - Size: int64(len(b)), - Typeflag: tar.TypeReg, - }, - CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { - n, err := w.Write(b) - return int64(n), err - }, - } - -} - -func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord { - index := ocispec.Index{ - Versioned: ocispecs.Versioned{ - SchemaVersion: 2, - }, - Manifests: manifests, - } - - b, err := json.Marshal(index) - if err != nil { - panic(err) - } - - return tarRecord{ - Header: &tar.Header{ - Name: "index.json", - Mode: 0644, - Size: int64(len(b)), - Typeflag: tar.TypeReg, - }, - CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { - n, err := w.Write(b) - return int64(n), err - }, - } -} - -func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error { - sort.Slice(records, func(i, j int) bool { - return records[i].Header.Name < records[j].Header.Name - }) - - for _, record := range records { - if err := tw.WriteHeader(record.Header); err != nil { - return err - } - if record.CopyTo != nil { - n, err := record.CopyTo(ctx, tw) - if err != nil { - return err - } - if n != record.Header.Size { - return errors.Errorf("unexpected copy size for %s", record.Header.Name) - } - } else if record.Header.Size > 0 { - return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name) - } - } - return nil -} diff --git a/import.go b/import.go index 9825f3167..87c87f6d5 100644 --- a/import.go +++ b/import.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "io" + "strings" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" @@ -130,16 +131,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt } for _, m := range idx.Manifests { - if ref := m.Annotations[ocispec.AnnotationRefName]; ref != "" { - if iopts.imageRefT != nil { - ref = iopts.imageRefT(ref) - } - if ref != "" { - imgs = append(imgs, images.Image{ - Name: ref, - Target: m, - }) - } + names := imageNames(m.Annotations, iopts.imageRefT) + for _, name := range names { + imgs = append(imgs, images.Image{ + Name: name, + Target: m, + }) } if iopts.dgstRefT != nil { ref := iopts.dgstRefT(m.Digest) @@ -178,3 +175,35 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt return imgs, nil } + +func imageNames(annotations map[string]string, ociCleanup func(string) string) []string { + var names []string + for k, v := range annotations { + if k == ocispec.AnnotationRefName { + if ociCleanup != nil { + v = ociCleanup(v) + } + if v != "" { + names = appendSorted(names, v) + } + } else if k == images.AnnotationImageName || strings.HasPrefix(k, images.AnnotationImageNamePrefix) { + names = appendSorted(names, v) + + } + } + return names +} + +func appendSorted(arr []string, s string) []string { + for i, c := range arr { + if s < c { + arr = append(arr, "") + copy(arr[i+1:], arr[i:]) + arr[i] = s + return arr + } else if s == c { + return arr + } + } + return append(arr, s) +} diff --git a/import_test.go b/import_test.go index 2b9894d29..b435f79a9 100644 --- a/import_test.go +++ b/import_test.go @@ -17,6 +17,7 @@ package containerd import ( + "bytes" "context" "encoding/json" "io" @@ -35,9 +36,9 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// TestOCIExportAndImport exports testImage as a tar stream, +// TestExportAndImport exports testImage as a tar stream, // and import the tar stream as a new image. -func TestOCIExportAndImport(t *testing.T) { +func TestExportAndImport(t *testing.T) { // TODO: support windows if testing.Short() || runtime.GOOS == "windows" { t.Skip() @@ -51,12 +52,13 @@ func TestOCIExportAndImport(t *testing.T) { } defer client.Close() - pulled, err := client.Fetch(ctx, testImage) + _, err = client.Fetch(ctx, testImage) if err != nil { t.Fatal(err) } - exported, err := client.Export(ctx, pulled.Target) + wb := bytes.NewBuffer(nil) + err = client.Export(ctx, wb, archive.WithAllPlatforms(), archive.WithImage(client.ImageService(), testImage)) if err != nil { t.Fatal(err) } @@ -64,12 +66,15 @@ func TestOCIExportAndImport(t *testing.T) { opts := []ImportOpt{ WithImageRefTranslator(archive.AddRefPrefix("foo/bar")), } - imgrecs, err := client.Import(ctx, exported, opts...) + imgrecs, err := client.Import(ctx, bytes.NewReader(wb.Bytes()), opts...) if err != nil { t.Fatalf("Import failed: %+v", err) } for _, imgrec := range imgrecs { + if imgrec.Name == testImage { + continue + } err = client.ImageService().Delete(ctx, imgrec.Name) if err != nil { t.Fatal(err)