Refactor image importer

Allow customization of reference creation.
Add option for digest references.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

images/importexport.go

@@ -27,7 +27,7 @@ import (
 // Importer is the interface for image importer.
 type Importer interface {
 	// Import imports an image from a tar stream.
-	Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error)
+	Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error)
 }
 
 // Exporter is the interface for image exporter.

images/oci/importer.go

@@ -19,114 +19,89 @@ package oci
 
 import (
 	"archive/tar"
+	"bytes"
 	"context"
-	"encoding/json"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"path"
 	"strings"
 
 	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/log"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
 // V1Importer implements OCI Image Spec v1.
-type V1Importer struct {
-	// ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs).
-	// This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355
-	ImageName string
-}
+type V1Importer struct{}
 
 var _ images.Importer = &V1Importer{}
 
 // Import implements Importer.
-func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) {
-	if oi.ImageName == "" {
-		return nil, errors.New("ImageName not set")
-	}
-	tr := tar.NewReader(reader)
-	var imgrecs []images.Image
-	foundIndexJSON := false
+func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) {
+	var (
+		desc ocispec.Descriptor
+		tr   = tar.NewReader(reader)
+	)
 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
-			return nil, err
+			return ocispec.Descriptor{}, err
 		}
 		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
+			log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
 			continue
 		}
 		hdrName := path.Clean(hdr.Name)
 		if hdrName == "index.json" {
-			if foundIndexJSON {
-				return nil, errors.New("duplicated index.json")
+			if desc.Digest != "" {
+				return ocispec.Descriptor{}, errors.New("duplicated index.json")
 			}
-			foundIndexJSON = true
-			imgrecs, err = onUntarIndexJSON(tr, oi.ImageName)
+			desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size)
 			if err != nil {
-				return nil, err
+				return ocispec.Descriptor{}, err
 			}
-			continue
-		}
-		if strings.HasPrefix(hdrName, "blobs/") {
+		} else if strings.HasPrefix(hdrName, "blobs/") {
 			if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil {
-				return nil, err
+				return ocispec.Descriptor{}, err
 			}
+		} else if hdrName == ocispec.ImageLayoutFile {
+			// TODO Validate
+		} else {
+			log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored")
 		}
 	}
-	if !foundIndexJSON {
-		return nil, errors.New("no index.json found")
+	if desc.Digest == "" {
+		return ocispec.Descriptor{}, errors.New("no index.json found")
 	}
-	for _, img := range imgrecs {
-		err := setGCRefContentLabels(ctx, store, img.Target)
-		if err != nil {
-			return imgrecs, err
-		}
-	}
-	// FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)?
-	return imgrecs, nil
+
+	return desc, nil
 }
 
-func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) {
+func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) {
 	b, err := ioutil.ReadAll(r)
 	if err != nil {
-		return nil, err
+		return ocispec.Descriptor{}, err
 	}
-	var idx ocispec.Index
-	if err := json.Unmarshal(b, &idx); err != nil {
-		return nil, err
+	desc := ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageIndex,
+		Digest:    digest.FromBytes(b),
+		Size:      size,
 	}
-	var imgrecs []images.Image
-	for _, m := range idx.Manifests {
-		ref, err := normalizeImageRef(imageName, m)
-		if err != nil {
-			return nil, err
-		}
-		imgrecs = append(imgrecs, images.Image{
-			Name:   ref,
-			Target: m,
-		})
+	if int64(len(b)) != size {
+		return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size)
 	}
-	return imgrecs, nil
-}
 
-func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) {
-	digest := manifest.Digest
-	if digest == "" {
-		return "", errors.Errorf("manifest with empty digest: %v", manifest)
+	if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil {
+		return ocispec.Descriptor{}, err
 	}
-	ociRef := manifest.Annotations[ocispec.AnnotationRefName]
-	if ociRef == "" {
-		return imageName + "@" + digest.String(), nil
-	}
-	return imageName + ":" + ociRef, nil
+
+	return desc, err
 }
 
 func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error {
@@ -140,65 +115,22 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name
 		return errors.Errorf("unsupported algorithm: %s", algo)
 	}
 	dgst := digest.NewDigestFromHex(algo.String(), split[2])
-	return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
+	return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
 }
 
-// GetChildrenDescriptors returns children blob descriptors for the following supported types:
-// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest
-// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex
-func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-	switch desc.MediaType {
-	case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
-		var manifest ocispec.Manifest
-		if err := json.NewDecoder(r).Decode(&manifest); err != nil {
-			return nil, err
-		}
-		return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil
-	case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
-		var index ocispec.Index
-		if err := json.NewDecoder(r).Decode(&index); err != nil {
-			return nil, err
-		}
-		return index.Manifests, nil
+// RefTranslator creates a reference using an OCI ref annotation,
+// which is mentioned in the spec as only a tag compontent,
+// concatenated with an image name
+func RefTranslator(prefix string) func(string) string {
+	return func(ref string) string {
+		return prefix + ":" + ref
 	}
-	return nil, nil
 }
 
-func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error {
-	info, err := store.Info(ctx, desc.Digest)
-	if err != nil {
-		if errdefs.IsNotFound(err) {
-			// when the archive is created from multi-arch image,
-			// it may contain only blobs for a certain platform.
-			// So ErrNotFound (on manifest list) is expected here.
-			return nil
-		}
-		return err
+// DigestTranslator creates a digest reference by adding the
+// digest to an image name
+func DigestTranslator(prefix string) func(digest.Digest) string {
+	return func(dgst digest.Digest) string {
+		return prefix + "@" + dgst.String()
 	}
-	ra, err := store.ReaderAt(ctx, desc)
-	if err != nil {
-		return err
-	}
-	defer ra.Close()
-	r := content.NewReader(ra)
-	children, err := GetChildrenDescriptors(r, desc)
-	if err != nil {
-		return err
-	}
-	if info.Labels == nil {
-		info.Labels = map[string]string{}
-	}
-	for i, child := range children {
-		// Note: child blob is not guaranteed to be written to the content store. (multi-arch)
-		info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String()
-	}
-	if _, err := store.Update(ctx, info, "labels"); err != nil {
-		return err
-	}
-	for _, child := range children {
-		if err := setGCRefContentLabels(ctx, store, child); err != nil {
-			return err
-		}
-	}
-	return nil
 }

images/oci/importer_test.go

@@ -1,53 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package oci
-
-import (
-	"testing"
-
-	"github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"gotest.tools/assert"
-)
-
-func TestNormalizeImageRef(t *testing.T) {
-	imageBaseName := "foo/bar"
-	for _, test := range []struct {
-		input  ocispec.Descriptor
-		expect string
-	}{
-		{
-			input: ocispec.Descriptor{
-				Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"),
-			},
-			expect: "foo/bar@sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77",
-		},
-		{
-			input: ocispec.Descriptor{
-				Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"),
-				Annotations: map[string]string{
-					ocispec.AnnotationRefName: "latest",
-				},
-			},
-			expect: "foo/bar:latest", // no @digest for simplicity
-		},
-	} {
-		normalized, err := normalizeImageRef(imageBaseName, test.input)
-		assert.NilError(t, err)
-		assert.Equal(t, test.expect, normalized)
-	}
-}