github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Refactor image importer

Browse Source 
Allow customization of reference creation.
Add option for digest references.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
This commit is contained in:
Derek McGowan 2018-09-06 12:03:17 -07:00
parent 05984a966d
commit f57c5cdefb
No known key found for this signature in database
GPG Key ID: F58C5D0A4405ACDB
6 changed files with 206 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
cmd/ctr/commands/images/import.go
View File

@ -20,9 +20,10 @@ import (
"fmt"
"io"
"os"
"time"
"github.com/containerd/containerd"
"github.com/containerd/containerd/cmd/ctr/commands"
"github.com/containerd/containerd/images"
oci "github.com/containerd/containerd/images/oci"
"github.com/containerd/containerd/log"
"github.com/urfave/cli"
@ -53,23 +54,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
Usage: "image format. See DESCRIPTION.",
},
cli.StringFlag{
Name: "oci-name",
Value: "unknown/unknown",
Usage: "prefix added to either oci.v1 ref annotation or digest",
Name: "prefix,oci-name",
Value: "",
Usage: "prefix image name for added images",
},
cli.BoolFlag{
Name: "digests",
Usage: "whether to create digest images",
},
// TODO(AkihiroSuda): support commands.LabelFlag (for all children objects)
}, commands.SnapshotterFlags...),
Action: func(context *cli.Context) error {
var (
in = context.Args().First()
imageImporter images.Importer
opts []containerd.ImportOpt
)
switch format := context.String("format"); format {
case "oci.v1":
imageImporter = &oci.V1Importer{
ImageName: context.String("oci-name"),
opts = append(opts, containerd.WithImporter(&oci.V1Importer{}))
prefix := context.String("prefix")
if prefix == "" {
prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02"))
}
opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix)))
if context.Bool("digests") {
opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix)))
}
default:
return fmt.Errorf("unknown format %s", format)
@ -90,20 +102,24 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
return err
}
}
imgs, err := client.Import(ctx, imageImporter, r)
imgs, err := client.Import(ctx, r, opts...)
closeErr := r.Close()
if err != nil {
return err
}
if err = r.Close(); err != nil {
return err
if closeErr != nil {
return closeErr
}
log.G(ctx).Debugf("unpacking %d images", len(imgs))
for _, img := range imgs {
// TODO: Allow configuration of the platform
image := containerd.NewImage(client, img)
// TODO: Show unpack status
fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest)
err = img.Unpack(ctx, context.String("snapshotter"))
fmt.Printf("unpacking %s (%s)...", img.Name, img.Target.Digest)
err = image.Unpack(ctx, context.String("snapshotter"))
if err != nil {
return err
}

2
images/importexport.go
View File

@ -27,7 +27,7 @@ import (
// Importer is the interface for image importer.
type Importer interface {
// Import imports an image from a tar stream.
Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error)
Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error)
}
// Exporter is the interface for image exporter.

168
images/oci/importer.go
View File

@ -19,114 +19,89 @@ package oci
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"path"
"strings"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/log"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
// V1Importer implements OCI Image Spec v1.
type V1Importer struct {
// ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs).
// This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355
ImageName string
}
type V1Importer struct{}
var _ images.Importer = &V1Importer{}
// Import implements Importer.
func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) {
if oi.ImageName == "" {
return nil, errors.New("ImageName not set")
}
tr := tar.NewReader(reader)
var imgrecs []images.Image
foundIndexJSON := false
func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) {
var (
desc ocispec.Descriptor
tr = tar.NewReader(reader)
)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
return ocispec.Descriptor{}, err
}
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
continue
}
hdrName := path.Clean(hdr.Name)
if hdrName == "index.json" {
if foundIndexJSON {
return nil, errors.New("duplicated index.json")
if desc.Digest != "" {
return ocispec.Descriptor{}, errors.New("duplicated index.json")
}
foundIndexJSON = true
imgrecs, err = onUntarIndexJSON(tr, oi.ImageName)
desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size)
if err != nil {
return nil, err
return ocispec.Descriptor{}, err
}
continue
}
if strings.HasPrefix(hdrName, "blobs/") {
} else if strings.HasPrefix(hdrName, "blobs/") {
if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil {
return nil, err
return ocispec.Descriptor{}, err
}
} else if hdrName == ocispec.ImageLayoutFile {
// TODO Validate
} else {
log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored")
}
}
}
if !foundIndexJSON {
return nil, errors.New("no index.json found")
}
for _, img := range imgrecs {
err := setGCRefContentLabels(ctx, store, img.Target)
if err != nil {
return imgrecs, err
}
}
// FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)?
return imgrecs, nil
if desc.Digest == "" {
return ocispec.Descriptor{}, errors.New("no index.json found")
}
func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) {
return desc, nil
}
func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) {
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
return ocispec.Descriptor{}, err
}
var idx ocispec.Index
if err := json.Unmarshal(b, &idx); err != nil {
return nil, err
desc := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageIndex,
Digest: digest.FromBytes(b),
Size: size,
}
var imgrecs []images.Image
for _, m := range idx.Manifests {
ref, err := normalizeImageRef(imageName, m)
if err != nil {
return nil, err
}
imgrecs = append(imgrecs, images.Image{
Name: ref,
Target: m,
})
}
return imgrecs, nil
if int64(len(b)) != size {
return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size)
}
func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) {
digest := manifest.Digest
if digest == "" {
return "", errors.Errorf("manifest with empty digest: %v", manifest)
if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil {
return ocispec.Descriptor{}, err
}
ociRef := manifest.Annotations[ocispec.AnnotationRefName]
if ociRef == "" {
return imageName + "@" + digest.String(), nil
}
return imageName + ":" + ociRef, nil
return desc, err
}
func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error {
@ -140,65 +115,22 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name
return errors.Errorf("unsupported algorithm: %s", algo)
}
dgst := digest.NewDigestFromHex(algo.String(), split[2])
return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
}
// GetChildrenDescriptors returns children blob descriptors for the following supported types:
// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest
// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex
func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
var manifest ocispec.Manifest
if err := json.NewDecoder(r).Decode(&manifest); err != nil {
return nil, err
// RefTranslator creates a reference using an OCI ref annotation,
// which is mentioned in the spec as only a tag compontent,
// concatenated with an image name
func RefTranslator(prefix string) func(string) string {
return func(ref string) string {
return prefix + ":" + ref
}
return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
var index ocispec.Index
if err := json.NewDecoder(r).Decode(&index); err != nil {
return nil, err
}
return index.Manifests, nil
}
return nil, nil
}
func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error {
info, err := store.Info(ctx, desc.Digest)
if err != nil {
if errdefs.IsNotFound(err) {
// when the archive is created from multi-arch image,
// it may contain only blobs for a certain platform.
// So ErrNotFound (on manifest list) is expected here.
return nil
}
return err
}
ra, err := store.ReaderAt(ctx, desc)
if err != nil {
return err
}
defer ra.Close()
r := content.NewReader(ra)
children, err := GetChildrenDescriptors(r, desc)
if err != nil {
return err
}
if info.Labels == nil {
info.Labels = map[string]string{}
}
for i, child := range children {
// Note: child blob is not guaranteed to be written to the content store. (multi-arch)
info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String()
}
if _, err := store.Update(ctx, info, "labels"); err != nil {
return err
}
for _, child := range children {
if err := setGCRefContentLabels(ctx, store, child); err != nil {
return err
// DigestTranslator creates a digest reference by adding the
// digest to an image name
func DigestTranslator(prefix string) func(digest.Digest) string {
return func(dgst digest.Digest) string {
return prefix + "@" + dgst.String()
}
}
return nil
}

53
images/oci/importer_test.go
View File

@ -1,53 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oci
import (
"testing"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"gotest.tools/assert"
)
func TestNormalizeImageRef(t *testing.T) {
imageBaseName := "foo/bar"
for _, test := range []struct {
input ocispec.Descriptor
expect string
}{
{
input: ocispec.Descriptor{
Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"),
},
expect: "foo/bar@sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77",
},
{
input: ocispec.Descriptor{
Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"),
Annotations: map[string]string{
ocispec.AnnotationRefName: "latest",
},
},
expect: "foo/bar:latest", // no @digest for simplicity
},
} {
normalized, err := normalizeImageRef(imageBaseName, test.input)
assert.NilError(t, err)
assert.Equal(t, test.expect, normalized)
}
}

143
import.go
View File

@ -18,36 +18,76 @@ package containerd
import (
"context"
"encoding/json"
"io"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/images/oci"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type importOpts struct {
indexName string
imageRefT func(string) string
dgstRefT func(digest.Digest) string
importer images.Importer
}
// ImportOpt allows the caller to specify import specific options
type ImportOpt func(c *importOpts) error
type ImportOpt func(*importOpts) error
func resolveImportOpt(opts ...ImportOpt) (importOpts, error) {
var iopts importOpts
for _, o := range opts {
if err := o(&iopts); err != nil {
return iopts, err
// WithImageRefTranslator is used to translate the index reference
// to an image reference for the image store.
func WithImageRefTranslator(f func(string) string) ImportOpt {
return func(c *importOpts) error {
c.imageRefT = f
return nil
}
}
return iopts, nil
// WithDigestRef is used to create digest images for each
// manifest in the index.
func WithDigestRef(f func(digest.Digest) string) ImportOpt {
return func(c *importOpts) error {
c.dgstRefT = f
return nil
}
}
// WithIndexName creates a tag pointing to the imported index
func WithIndexName(name string) ImportOpt {
return func(c *importOpts) error {
c.indexName = name
return nil
}
}
// WithImporter sets the importer to use for converting
// the read stream into an OCI Index.
func WithImporter(importer images.Importer) ImportOpt {
return func(c *importOpts) error {
c.importer = importer
return nil
}
}
// Import imports an image from a Tar stream using reader.
// Caller needs to specify importer. Future version may use oci.v1 as the default.
// Note that unreferrenced blobs may be imported to the content store as well.
func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) {
_, err := resolveImportOpt(opts...) // unused now
if err != nil {
func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt) ([]images.Image, error) {
var iopts importOpts
for _, o := range opts {
if err := o(&iopts); err != nil {
return nil, err
}
}
if iopts.importer == nil {
iopts.importer = &oci.V1Importer{}
}
ctx, done, err := c.WithLease(ctx)
if err != nil {
@ -55,31 +95,86 @@ func (c *Client) Import(ctx context.Context, importer images.Importer, reader io
}
defer done(ctx)
imgrecs, err := importer.Import(ctx, c.ContentStore(), reader)
index, err := iopts.importer.Import(ctx, c.ContentStore(), reader)
if err != nil {
// is.Update() is not called on error
return nil, err
}
is := c.ImageService()
var images []Image
for _, imgrec := range imgrecs {
if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
var (
imgs []images.Image
cs = c.ContentStore()
is = c.ImageService()
)
if iopts.indexName != "" {
imgs = append(imgs, images.Image{
Name: iopts.indexName,
Target: index,
})
}
var handler images.HandlerFunc
handler = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
// Only save images at top level
if desc.Digest != index.Digest {
return images.Children(ctx, cs, desc)
}
p, err := content.ReadBlob(ctx, cs, desc)
if err != nil {
return nil, err
}
var idx ocispec.Index
if err := json.Unmarshal(p, &idx); err != nil {
return nil, err
}
for _, m := range idx.Manifests {
if ref := m.Annotations[ocispec.AnnotationRefName]; ref != "" {
if iopts.imageRefT != nil {
ref = iopts.imageRefT(ref)
}
if ref != "" {
imgs = append(imgs, images.Image{
Name: ref,
Target: m,
})
}
}
if iopts.dgstRefT != nil {
ref := iopts.dgstRefT(m.Digest)
if ref != "" {
imgs = append(imgs, images.Image{
Name: ref,
Target: m,
})
}
}
}
return idx.Manifests, nil
}
handler = images.SetChildrenLabels(cs, handler)
if err := images.Walk(ctx, handler, index); err != nil {
return nil, err
}
for i := range imgs {
img, err := is.Update(ctx, imgs[i], "target")
if err != nil {
if !errdefs.IsNotFound(err) {
return nil, err
}
created, err := is.Create(ctx, imgrec)
img, err = is.Create(ctx, imgs[i])
if err != nil {
return nil, err
}
imgrec = created
} else {
imgrec = updated
}
imgs[i] = img
}
images = append(images, NewImage(c, imgrec))
}
return images, nil
return imgs, nil
}

8
import_test.go
View File

@ -49,13 +49,17 @@ func TestOCIExportAndImport(t *testing.T) {
t.Fatal(err)
}
imgrecs, err := client.Import(ctx, &oci.V1Importer{ImageName: "foo/bar:"}, exported)
opts := []ImportOpt{
WithImporter(&oci.V1Importer{}),
WithImageRefTranslator(oci.RefTranslator("foo/bar")),
}
imgrecs, err := client.Import(ctx, exported, opts...)
if err != nil {
t.Fatal(err)
}
for _, imgrec := range imgrecs {
err = client.ImageService().Delete(ctx, imgrec.Name())
err = client.ImageService().Delete(ctx, imgrec.Name)
if err != nil {
t.Fatal(err)
}