github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #7964 from dmcgowan/transfer-image-store-references

Browse Source 
[transfer] update imagestore interface to support multiple references
This commit is contained in:
Akihiro Suda
2023-02-14 11:22:27 +09:00
committed by GitHub
parent 458fd48720 081601f521
commit 4e2eb8ba4e
10 changed files with 996 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
pkg/transfer/image/imagestore.go
View File

@@ -48,15 +48,42 @@ type Store struct {
labelMap func(ocispec.Descriptor) []string
manifestLimit int
//import image options
namePrefix string
checkPrefix bool
digestRefs bool
alwaysDigest bool
// extraReferences are used to store or lookup multiple references
extraReferences []Reference
unpacks []UnpackConfiguration
}
// Reference is used to create or find a reference for an image
type Reference struct {
Name string
// IsPrefix determines whether the Name should be considered
// a prefix (without tag or digest).
// For lookup, this may allow matching multiple tags.
// For store, this must have a tag or digest added.
IsPrefix bool
// AllowOverwrite allows overwriting or ignoring the name if
// another reference is provided (such as through an annotation).
// Only used if IsPrefix is true.
AllowOverwrite bool
// AddDigest adds the manifest digest to the reference.
// For lookup, this allows matching tags with any digest.
// For store, this allows adding the digest to the name.
// Only used if IsPrefix is true.
AddDigest bool
// SkipNamedDigest only considers digest references which do not
// have a non-digested named reference.
// For lookup, this will deduplicate digest references when there is a named match.
// For store, this only adds this digest reference when there is no matching full
// name reference from the prefix.
// Only used if IsPrefix is true.
SkipNamedDigest bool
}
// UnpackConfiguration specifies the platform and snapshotter to use for resolving
// the unpack Platform, if snapshotter is not specified the platform default will
// be used.
@@ -93,22 +120,51 @@ func WithAllMetadata(s *Store) {
s.allMetadata = true
}
// WithNamePrefix sets the name prefix for imported images, if
// check is enabled, then only images with the prefix are stored.
func WithNamePrefix(prefix string, check bool) StoreOpt {
// WithNamedPrefix uses a named prefix to references images which only have a tag name
// reference in the annotation or check full references annotations against. Images
// with no reference resolved from matching annotations will not be stored.
// - name: image name prefix to append a tag to or check full name references with
// - allowOverwrite: allows the tag to be overwritten by full name reference inside
// the image which does not have name as the prefix
func WithNamedPrefix(name string, allowOverwrite bool) StoreOpt {
ref := Reference{
Name: name,
IsPrefix: true,
AllowOverwrite: allowOverwrite,
}
return func(s *Store) {
s.namePrefix = prefix
s.checkPrefix = check
s.extraReferences = append(s.extraReferences, ref)
}
}
// WithDigestRefs sets digest refs for imported images, if
// always is enabled, then digest refs are added even if a
// non-digest image name is added for the same image.
func WithDigestRefs(always bool) StoreOpt {
// WithNamedPrefix uses a named prefix to references images which only have a tag name
// reference in the annotation or check full references annotations against and
// additionally may add a digest reference. Images with no references resolved
// from matching annotations may be stored by digest.
// - name: image name prefix to append a tag to or check full name references with
// - allowOverwrite: allows the tag to be overwritten by full name reference inside
// the image which does not have name as the prefix
// - skipNamed: is set if no digest reference should be created if a named reference
// is successfully resolved from the annotations.
func WithDigestRef(name string, allowOverwrite bool, skipNamed bool) StoreOpt {
ref := Reference{
Name: name,
IsPrefix: true,
AllowOverwrite: allowOverwrite,
AddDigest: true,
SkipNamedDigest: skipNamed,
}
return func(s *Store) {
s.digestRefs = true
s.alwaysDigest = always
s.extraReferences = append(s.extraReferences, ref)
}
}
func WithExtraReference(name string) StoreOpt {
ref := Reference{
Name: name,
}
return func(s *Store) {
s.extraReferences = append(s.extraReferences, ref)
}
}
@@ -163,64 +219,114 @@ func (is *Store) ImageFilter(h images.HandlerFunc, cs content.Store) images.Hand
return h
}
func (is *Store) Store(ctx context.Context, desc ocispec.Descriptor, store images.Store) (images.Image, error) {
img := images.Image{
Name: is.imageName,
Target: desc,
Labels: is.imageLabels,
}
func (is *Store) Store(ctx context.Context, desc ocispec.Descriptor, store images.Store) ([]images.Image, error) {
var imgs []images.Image
// Handle imported image names
if refType, ok := desc.Annotations["io.containerd.import.ref-type"]; ok {
var nameT func(string) string
if is.checkPrefix {
nameT = archive.FilterRefPrefix(is.namePrefix)
} else {
nameT = archive.AddRefPrefix(is.namePrefix)
}
name := imageName(desc.Annotations, nameT)
switch refType {
case "name":
if name == "" {
return images.Image{}, fmt.Errorf("no image name: %w", errdefs.ErrNotFound)
// If import ref type, store references from annotation or prefix
if refSource, ok := desc.Annotations["io.containerd.import.ref-source"]; ok {
switch refSource {
case "annotation":
for _, ref := range is.extraReferences {
// Only use prefix references for annotation matching
if !ref.IsPrefix {
continue
}
var nameT func(string) string
if ref.AllowOverwrite {
nameT = archive.AddRefPrefix(ref.Name)
} else {
nameT = archive.FilterRefPrefix(ref.Name)
}
name := imageName(desc.Annotations, nameT)
if name == "" {
// If digested, add digest reference
if ref.AddDigest {
imgs = append(imgs, images.Image{
Name: fmt.Sprintf("%s@%s", ref.Name, desc.Digest),
Target: desc,
Labels: is.imageLabels,
})
}
continue
}
imgs = append(imgs, images.Image{
Name: name,
Target: desc,
Labels: is.imageLabels,
})
// If a named reference was found and SkipNamedDigest is true, do
// not use this reference
if ref.AddDigest && !ref.SkipNamedDigest {
imgs = append(imgs, images.Image{
Name: fmt.Sprintf("%s@%s", ref.Name, desc.Digest),
Target: desc,
Labels: is.imageLabels,
})
}
}
img.Name = name
case "digest":
if !is.digestRefs || (!is.alwaysDigest && name != "") {
return images.Image{}, fmt.Errorf("no digest refs: %w", errdefs.ErrNotFound)
}
img.Name = fmt.Sprintf("%s@%s", is.namePrefix, desc.Digest)
default:
return images.Image{}, fmt.Errorf("ref type not supported: %w", errdefs.ErrInvalidArgument)
return nil, fmt.Errorf("ref source not supported: %w", errdefs.ErrInvalidArgument)
}
delete(desc.Annotations, "io.containerd.import.ref-source")
} else {
if is.imageName != "" {
imgs = append(imgs, images.Image{
Name: is.imageName,
Target: desc,
Labels: is.imageLabels,
})
}
// If extra references, store all complete references (skip prefixes)
for _, ref := range is.extraReferences {
if ref.IsPrefix {
continue
}
name := ref.Name
if ref.AddDigest {
name = fmt.Sprintf("%s@%s", name, desc.Digest)
}
imgs = append(imgs, images.Image{
Name: name,
Target: desc,
Labels: is.imageLabels,
})
}
delete(desc.Annotations, "io.containerd.import.ref-type")
} else if img.Name == "" {
// No valid image combination found
return images.Image{}, fmt.Errorf("no image name found: %w", errdefs.ErrNotFound)
}
for {
if created, err := store.Create(ctx, img); err != nil {
if len(imgs) == 0 {
return nil, fmt.Errorf("no image name found: %w", errdefs.ErrNotFound)
}
for i := 0; i < len(imgs); {
if created, err := store.Create(ctx, imgs[i]); err != nil {
if !errdefs.IsAlreadyExists(err) {
return images.Image{}, err
return nil, err
}
updated, err := store.Update(ctx, img)
updated, err := store.Update(ctx, imgs[i])
if err != nil {
// if image was removed, try create again
if errdefs.IsNotFound(err) {
// Keep trying same image
continue
}
return images.Image{}, err
return nil, err
}
img = updated
imgs[i] = updated
} else {
img = created
imgs[i] = created
}
return img, nil
i++
}
return imgs, nil
}
func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, error) {
@@ -239,16 +345,13 @@ func (is *Store) UnpackPlatforms() []unpack.Platform {
func (is *Store) MarshalAny(context.Context, streaming.StreamCreator) (typeurl.Any, error) {
//unpack.Platform
s := &transfertypes.ImageStore{
Name: is.imageName,
Labels: is.imageLabels,
ManifestLimit: uint32(is.manifestLimit),
AllMetadata: is.allMetadata,
Platforms: platformsToProto(is.platforms),
Prefix: is.namePrefix,
CheckPrefix: is.checkPrefix,
DigestRefs: is.digestRefs,
AlwaysDigest: is.alwaysDigest,
Unpacks: unpackToProto(is.unpacks),
Name: is.imageName,
Labels: is.imageLabels,
ManifestLimit: uint32(is.manifestLimit),
AllMetadata: is.allMetadata,
Platforms: platformsToProto(is.platforms),
ExtraReferences: referencesToProto(is.extraReferences),
Unpacks: unpackToProto(is.unpacks),
}
return typeurl.MarshalAny(s)
}
@@ -264,10 +367,7 @@ func (is *Store) UnmarshalAny(ctx context.Context, sm streaming.StreamGetter, a
is.manifestLimit = int(s.ManifestLimit)
is.allMetadata = s.AllMetadata
is.platforms = platformFromProto(s.Platforms)
is.namePrefix = s.Prefix
is.checkPrefix = s.CheckPrefix
is.digestRefs = s.DigestRefs
is.alwaysDigest = s.AlwaysDigest
is.extraReferences = referencesFromProto(s.ExtraReferences)
is.unpacks = unpackFromProto(s.Unpacks)
return nil
@@ -297,6 +397,33 @@ func platformFromProto(platforms []*types.Platform) []ocispec.Platform {
return op
}
func referencesToProto(references []Reference) []*transfertypes.ImageReference {
ir := make([]*transfertypes.ImageReference, len(references))
for i := range references {
r := transfertypes.ImageReference{
Name: references[i].Name,
IsPrefix: references[i].IsPrefix,
AllowOverwrite: references[i].AllowOverwrite,
AddDigest: references[i].AddDigest,
SkipNamedDigest: references[i].SkipNamedDigest,
}
ir[i] = &r
}
return ir
}
func referencesFromProto(references []*transfertypes.ImageReference) []Reference {
or := make([]Reference, len(references))
for i := range references {
or[i].Name = references[i].Name
or[i].IsPrefix = references[i].IsPrefix
or[i].AllowOverwrite = references[i].AllowOverwrite
or[i].AddDigest = references[i].AddDigest
or[i].SkipNamedDigest = references[i].SkipNamedDigest
}
return or
}
func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguration {
auc := make([]*transfertypes.UnpackConfiguration, len(uc))
for i := range uc {
@@ -326,15 +453,23 @@ func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []UnpackConfigura
return uc
}
func imageName(annotations map[string]string, ociCleanup func(string) string) string {
func imageName(annotations map[string]string, cleanup func(string) string) string {
name := annotations[images.AnnotationImageName]
if name != "" {
if cleanup != nil {
// containerd reference name should be full reference and not
// modified, if it is incomplete or does not match a specified
// prefix, do not use the reference
if cleanName := cleanup(name); cleanName != name {
name = ""
}
}
return name
}
name = annotations[ocispec.AnnotationRefName]
if name != "" {
if ociCleanup != nil {
name = ociCleanup(name)
if cleanup != nil {
name = cleanup(name)
}
}
return name

275
pkg/transfer/image/imagestore_test.go Normal file
View File

@@ -0,0 +1,275 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package image
import (
"context"
"errors"
"testing"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
func TestStore(t *testing.T) {
for _, testCase := range []struct {
Name string
ImageStore *Store
// Annotations are the different references annotations to run the test with,
// the possible values:
// - "OCI": Uses the OCI defined annotation "org.opencontainers.image.ref.name"
// This annotation may be a full reference or tag only
// - "containerd": Uses the containerd defined annotation "io.containerd.image.name"
// This annotation is always a full reference as used by containerd
// - "Annotation": Sets the annotation flag but does not set a reference annotation
// Use this case to test the default where no reference is provided
// - "NoAnnotation": Does not set the annotation flag
// Use this case to test storing of the index images by reference
Annotations []string
ImageName string
Images []string
Err error
}{
{
Name: "Prefix",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "registry.test/image",
IsPrefix: true,
},
},
},
Annotations: []string{"OCI", "containerd"},
ImageName: "registry.test/image:latest",
Images: []string{"registry.test/image:latest"},
},
{
Name: "Overwrite",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "placeholder",
IsPrefix: true,
AllowOverwrite: true,
},
},
},
Annotations: []string{"OCI", "containerd"},
ImageName: "registry.test/image:latest",
Images: []string{"registry.test/image:latest"},
},
{
Name: "TagOnly",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "registry.test/image",
IsPrefix: true,
},
},
},
Annotations: []string{"OCI"},
ImageName: "latest",
Images: []string{"registry.test/image:latest"},
},
{
Name: "AddDigest",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "registry.test/base",
IsPrefix: true,
AddDigest: true,
},
},
},
Annotations: []string{"Annotation"},
Images: []string{"registry.test/base@"},
},
{
Name: "NameAndDigest",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "registry.test/base",
IsPrefix: true,
AddDigest: true,
},
},
},
Annotations: []string{"OCI"},
ImageName: "latest",
Images: []string{"registry.test/base:latest", "registry.test/base@"},
},
{
Name: "NameSkipDigest",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "registry.test/base",
IsPrefix: true,
AddDigest: true,
SkipNamedDigest: true,
},
},
},
Annotations: []string{"OCI"},
ImageName: "latest",
Images: []string{"registry.test/base:latest"},
},
{
Name: "OverwriteNameDigest",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "base-name",
IsPrefix: true,
AllowOverwrite: true,
AddDigest: true,
},
},
},
Annotations: []string{"OCI", "containerd"},
ImageName: "registry.test/base:latest",
Images: []string{"registry.test/base:latest", "base-name@"},
},
{
Name: "OverwriteNameSkipDigest",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "base-name",
IsPrefix: true,
AllowOverwrite: true,
AddDigest: true,
SkipNamedDigest: true,
},
},
},
Annotations: []string{"OCI", "containerd"},
ImageName: "registry.test/base:latest",
Images: []string{"registry.test/base:latest"},
},
{
Name: "ReferenceNotFound",
ImageStore: &Store{
extraReferences: []Reference{
{
Name: "registry.test/image",
IsPrefix: true,
},
},
},
Annotations: []string{"OCI", "containerd"},
ImageName: "registry.test/base:latest",
Err: errdefs.ErrNotFound,
},
{
Name: "NoReference",
ImageStore: &Store{},
Annotations: []string{"Annotation", "NoAnnotation"},
Err: errdefs.ErrNotFound,
},
{
Name: "ImageName",
ImageStore: &Store{
imageName: "registry.test/index:latest",
},
Annotations: []string{"NoAnnotation"},
Images: []string{"registry.test/index:latest"},
},
} {
testCase := testCase
for _, a := range testCase.Annotations {
name := testCase.Name + "_" + a
dgst := digest.Canonical.FromString(name)
desc := ocispec.Descriptor{
Digest: dgst,
Annotations: map[string]string{},
}
expected := make([]string, len(testCase.Images))
for i, img := range testCase.Images {
if img[len(img)-1] == '@' {
img = img + dgst.String()
}
expected[i] = img
}
switch a {
case "containerd":
desc.Annotations["io.containerd.import.ref-source"] = "annotation"
desc.Annotations[images.AnnotationImageName] = testCase.ImageName
case "OCI":
desc.Annotations["io.containerd.import.ref-source"] = "annotation"
desc.Annotations[ocispec.AnnotationRefName] = testCase.ImageName
case "Annotation":
desc.Annotations["io.containerd.import.ref-source"] = "annotation"
}
t.Run(name, func(t *testing.T) {
imgs, err := testCase.ImageStore.Store(context.Background(), desc, nopImageStore{})
if err != nil {
if testCase.Err == nil {
t.Fatal(err)
}
if !errors.Is(err, testCase.Err) {
t.Fatalf("unexpected error %v: expeceted %v", err, testCase.Err)
}
return
} else if testCase.Err != nil {
t.Fatalf("succeeded but expected error: %v", testCase.Err)
}
if len(imgs) != len(expected) {
t.Fatalf("mismatched array length\nexpected:\n\t%v\nactual\n\t%v", expected, imgs)
}
for i, name := range expected {
if imgs[i].Name != name {
t.Fatalf("wrong image name %q, expected %q", imgs[i].Name, name)
}
if imgs[i].Target.Digest != dgst {
t.Fatalf("wrong image digest %s, expected %s", imgs[i].Target.Digest, dgst)
}
}
})
}
}
}
type nopImageStore struct{}
func (nopImageStore) Get(ctx context.Context, name string) (images.Image, error) {
return images.Image{}, errdefs.ErrNotFound
}
func (nopImageStore) List(ctx context.Context, filters ...string) ([]images.Image, error) {
return nil, nil
}
func (nopImageStore) Create(ctx context.Context, image images.Image) (images.Image, error) {
return image, nil
}
func (nopImageStore) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) {
return image, nil
}
func (nopImageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error {
return nil
}

25
pkg/transfer/local/import.go
View File

@@ -23,6 +23,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/transfer"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@@ -67,14 +68,8 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
}
for _, m := range idx.Manifests {
m1 := m
m1.Annotations = mergeMap(m.Annotations, map[string]string{"io.containerd.import.ref-type": "name"})
descriptors = append(descriptors, m1)
// If add digest references, add twice
m2 := m
m2.Annotations = mergeMap(m.Annotations, map[string]string{"io.containerd.import.ref-type": "digest"})
descriptors = append(descriptors, m2)
m.Annotations = mergeMap(m.Annotations, map[string]string{"io.containerd.import.ref-source": "annotation"})
descriptors = append(descriptors, m)
}
return idx.Manifests, nil
@@ -85,23 +80,27 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
}
if err := images.WalkNotEmpty(ctx, handler, index); err != nil {
// TODO: Handle Not Empty as a special case on the input
return err
}
for _, desc := range descriptors {
img, err := is.Store(ctx, desc, ts.images)
imgs, err := is.Store(ctx, desc, ts.images)
if err != nil {
if errdefs.IsNotFound(err) {
log.G(ctx).Infof("No images store for %s", desc.Digest)
continue
}
return err
}
if tops.Progress != nil {
tops.Progress(transfer.Progress{
Event: "saved",
Name: img.Name,
})
for _, img := range imgs {
tops.Progress(transfer.Progress{
Event: "saved",
Name: img.Name,
})
}
}
}

13
pkg/transfer/local/pull.go
View File

@@ -198,17 +198,18 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch
}
}
img, err := is.Store(ctx, desc, ts.images)
imgs, err := is.Store(ctx, desc, ts.images)
if err != nil {
return err
}
if tops.Progress != nil {
tops.Progress(transfer.Progress{
Event: "saved",
Name: img.Name,
//Digest: img.Target.Digest.String(),
})
for _, img := range imgs {
tops.Progress(transfer.Progress{
Event: "saved",
Name: img.Name,
})
}
}
if tops.Progress != nil {

7
pkg/transfer/transfer.go
View File

@@ -57,10 +57,11 @@ type ImageFilterer interface {
ImageFilter(images.HandlerFunc, content.Store) images.HandlerFunc
}
// ImageStorer is a type which is capable of storing an image to
// for a provided descriptor
// ImageStorer is a type which is capable of storing images for
// the provided descriptor. The descriptor may be any type of manifest
// including an index with multiple image references.
type ImageStorer interface {
Store(context.Context, ocispec.Descriptor, images.Store) (images.Image, error)
Store(context.Context, ocispec.Descriptor, images.Store) ([]images.Image, error)
}
// ImageGetter is type which returns an image from an image store