github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #7944 from adisky/new-pinned-image

Browse Source 
CRI Pinned image support
This commit is contained in:
Derek McGowan 2023-06-10 22:29:34 -07:00 committed by GitHub
commit dd5e9f6538
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 235 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
integration/containerd_image_test.go
View File

@ -207,3 +207,21 @@ func TestContainerdImageInOtherNamespaces(t *testing.T) {
}
assert.NoError(t, Consistently(checkImage, 100*time.Millisecond, time.Second))
}
func TestContainerdSandboxImage(t *testing.T) {
var pauseImage = images.Get(images.Pause)
ctx := context.Background()
t.Log("make sure the pause image exist")
pauseImg, err := containerdClient.GetImage(ctx, pauseImage)
require.NoError(t, err)
t.Log("ensure correct labels are set on pause image")
assert.Equal(t, pauseImg.Labels()["io.cri-containerd.pinned"], "pinned")
t.Log("pause image should be seen by cri plugin")
pimg, err := imageService.ImageStatus(&runtime.ImageSpec{Image: pauseImage})
require.NoError(t, err)
require.NotNil(t, pimg)
t.Log("verify pinned field is set for pause image")
assert.True(t, pimg.Pinned)
}

30
pkg/cri/labels/labels.go Normal file
View File

@ -0,0 +1,30 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package labels
const (
// criContainerdPrefix is common prefix for cri-containerd
criContainerdPrefix = "io.cri-containerd"
// ImageLabelKey is the label key indicating the image is managed by cri plugin.
ImageLabelKey = criContainerdPrefix + ".image"
// ImageLabelValue is the label value indicating the image is managed by cri plugin.
ImageLabelValue = "managed"
// PinnedImageLabelKey is the label value indicating the image is pinned.
PinnedImageLabelKey = criContainerdPrefix + ".pinned"
// PinnedImageLabelValue is the label value indicating the image is pinned.
PinnedImageLabelValue = "pinned"
)

42
pkg/cri/sbserver/images/image_pull.go
View File

@ -45,6 +45,7 @@ import (
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
crilabels "github.com/containerd/containerd/pkg/cri/labels"
snpkg "github.com/containerd/containerd/pkg/snapshotters"
distribution "github.com/containerd/containerd/reference/docker"
"github.com/containerd/containerd/remotes/docker"
@ -155,12 +156,14 @@ func (c *CRIImageService) PullImage(ctx context.Context, r *runtime.PullImageReq
tracing.Attribute("snapshotter.name", snapshotter),
)
labels := c.getLabels(ctx, ref)
pullOpts := []containerd.RemoteOpt{
containerd.WithSchema1Conversion, //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
containerd.WithResolver(resolver),
containerd.WithPullSnapshotter(snapshotter),
containerd.WithPullUnpack,
containerd.WithPullLabel(imageLabelKey, imageLabelValue),
containerd.WithPullLabels(labels),
containerd.WithMaxConcurrentDownloads(c.config.MaxConcurrentDownloads),
containerd.WithImageHandler(imageHandler),
containerd.WithUnpackOpts([]containerd.UnpackOpt{
@ -199,7 +202,7 @@ func (c *CRIImageService) PullImage(ctx context.Context, r *runtime.PullImageReq
if r == "" {
continue
}
if err := c.createImageReference(ctx, r, image.Target()); err != nil {
if err := c.createImageReference(ctx, r, image.Target(), labels); err != nil {
return nil, fmt.Errorf("failed to create image reference %q: %w", r, err)
}
// Update image store to reflect the newest state in containerd.
@ -283,12 +286,12 @@ func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) {
// Note that because create and update are not finished in one transaction, there could be race. E.g.
// the image reference is deleted by someone else after create returns already exists, but before update
// happens.
func (c *CRIImageService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor) error {
func (c *CRIImageService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor, labels map[string]string) error {
img := containerdimages.Image{
Name: name,
Target: desc,
// Add a label to indicate that the image is managed by the cri plugin.
Labels: map[string]string{imageLabelKey: imageLabelValue},
Labels: labels,
}
// TODO(random-liu): Figure out which is the more performant sequence create then update or
// update then create.
@ -296,14 +299,32 @@ func (c *CRIImageService) createImageReference(ctx context.Context, name string,
if err == nil || !errdefs.IsAlreadyExists(err) {
return err
}
if oldImg.Target.Digest == img.Target.Digest && oldImg.Labels[imageLabelKey] == imageLabelValue {
if oldImg.Target.Digest == img.Target.Digest && oldImg.Labels[crilabels.ImageLabelKey] == labels[crilabels.ImageLabelKey] {
return nil
}
_, err = c.client.ImageService().Update(ctx, img, "target", "labels."+imageLabelKey)
_, err = c.client.ImageService().Update(ctx, img, "target", "labels."+crilabels.ImageLabelKey)
return err
}
// UpdateImage updates image store to reflect the newest state of an image reference
// getLabels get image labels to be added on CRI image
func (c *CRIImageService) getLabels(ctx context.Context, name string) map[string]string {
labels := map[string]string{crilabels.ImageLabelKey: crilabels.ImageLabelValue}
configSandboxImage := c.config.SandboxImage
// parse sandbox image
sandboxNamedRef, err := distribution.ParseDockerRef(configSandboxImage)
if err != nil {
log.G(ctx).Errorf("failed to parse sandbox image from config %s", sandboxNamedRef)
return nil
}
sandboxRef := sandboxNamedRef.String()
// Adding pinned image label to sandbox image
if sandboxRef == name {
labels[crilabels.PinnedImageLabelKey] = crilabels.PinnedImageLabelValue
}
return labels
}
// updateImage updates image store to reflect the newest state of an image reference
// in containerd. If the reference is not managed by the cri plugin, the function also
// generates necessary metadata for the image and make it managed.
func (c *CRIImageService) UpdateImage(ctx context.Context, r string) error {
@ -311,7 +332,7 @@ func (c *CRIImageService) UpdateImage(ctx context.Context, r string) error {
if err != nil && !errdefs.IsNotFound(err) {
return fmt.Errorf("get image by reference: %w", err)
}
if err == nil && img.Labels()[imageLabelKey] != imageLabelValue {
if err == nil && img.Labels()[crilabels.ImageLabelKey] != crilabels.ImageLabelValue {
// Make sure the image has the image id as its unique
// identifier that references the image in its lifetime.
configDesc, err := img.Config(ctx)
@ -319,14 +340,15 @@ func (c *CRIImageService) UpdateImage(ctx context.Context, r string) error {
return fmt.Errorf("get image id: %w", err)
}
id := configDesc.Digest.String()
if err := c.createImageReference(ctx, id, img.Target()); err != nil {
labels := c.getLabels(ctx, id)
if err := c.createImageReference(ctx, id, img.Target(), labels); err != nil {
return fmt.Errorf("create image id reference %q: %w", id, err)
}
if err := c.imageStore.Update(ctx, id); err != nil {
return fmt.Errorf("update image store for %q: %w", id, err)
}
// The image id is ready, add the label to mark the image as managed.
if err := c.createImageReference(ctx, r, img.Target()); err != nil {
if err := c.createImageReference(ctx, r, img.Target(), labels); err != nil {
return fmt.Errorf("create managed label: %w", err)
}
}

55
pkg/cri/sbserver/images/image_pull_test.go
View File

@ -29,6 +29,7 @@ import (
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
"github.com/containerd/containerd/pkg/cri/labels"
)
func TestParseAuth(t *testing.T) {
@ -481,3 +482,57 @@ func TestGetRepoDigestAndTag(t *testing.T) {
})
}
}
func TestImageGetLabels(t *testing.T) {
criService := newTestCRIService()
tests := []struct {
name string
expectedLabel map[string]string
configSandboxImage string
pullImageName string
}{
{
name: "pinned image labels should get added on sandbox image",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause:3.9",
pullImageName: "k8s.gcr.io/pause:3.9",
},
{
name: "pinned image labels should get added on sandbox image without tag",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause",
pullImageName: "k8s.gcr.io/pause:latest",
},
{
name: "pinned image labels should get added on sandbox image specified with tag and digest both",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause:3.9@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
},
{
name: "pinned image labels should get added on sandbox image specified with digest",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
},
{
name: "pinned image labels should not get added on other image",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause:3.9",
pullImageName: "k8s.gcr.io/random:latest",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
criService.config.SandboxImage = tt.configSandboxImage
labels := criService.getLabels(context.Background(), tt.pullImageName)
assert.Equal(t, tt.expectedLabel, labels)
})
}
}

1
pkg/cri/sbserver/images/image_status.go
View File

@ -71,6 +71,7 @@ func toCRIImage(image imagestore.Image) *runtime.Image {
RepoTags: repoTags,
RepoDigests: repoDigests,
Size_: uint64(image.Size),
Pinned: image.Pinned,
}
uid, username := getUserFromImage(image.ImageSpec.Config.User)
if uid != nil {

7
pkg/cri/sbserver/images/service.go
View File

@ -32,13 +32,6 @@ import (
"github.com/sirupsen/logrus"
)
const (
// imageLabelKey is the label key indicating the image is managed by cri plugin.
imageLabelKey = "io.cri-containerd.image"
// imageLabelValue is the label value indicating the image is managed by cri plugin.
imageLabelValue = "managed"
)
type CRIImageService struct {
// config contains all configurations.
config criconfig.Config

4
pkg/cri/server/helpers.go
View File

@ -80,10 +80,6 @@ const (
containerKindSandbox = "sandbox"
// containerKindContainer is a label value indicating container is application container
containerKindContainer = "container"
// imageLabelKey is the label key indicating the image is managed by cri plugin.
imageLabelKey = criContainerdPrefix + ".image"
// imageLabelValue is the label value indicating the image is managed by cri plugin.
imageLabelValue = "managed"
// sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest
sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata"
// containerMetadataExtension is an extension name that identify metadata of container in CreateContainerRequest

41
pkg/cri/server/image_pull.go
View File

@ -43,6 +43,7 @@ import (
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
crilabels "github.com/containerd/containerd/pkg/cri/labels"
snpkg "github.com/containerd/containerd/pkg/snapshotters"
distribution "github.com/containerd/containerd/reference/docker"
"github.com/containerd/containerd/remotes/docker"
@ -152,12 +153,15 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
tracing.Attribute("image.ref", ref),
tracing.Attribute("snapshotter.name", snapshotter),
)
labels := c.getLabels(ctx, ref)
pullOpts := []containerd.RemoteOpt{
containerd.WithSchema1Conversion, //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
containerd.WithResolver(resolver),
containerd.WithPullSnapshotter(snapshotter),
containerd.WithPullUnpack,
containerd.WithPullLabel(imageLabelKey, imageLabelValue),
containerd.WithPullLabels(labels),
containerd.WithMaxConcurrentDownloads(c.config.MaxConcurrentDownloads),
containerd.WithImageHandler(imageHandler),
containerd.WithUnpackOpts([]containerd.UnpackOpt{
@ -196,7 +200,7 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
if r == "" {
continue
}
if err := c.createImageReference(ctx, r, image.Target()); err != nil {
if err := c.createImageReference(ctx, r, image.Target(), labels); err != nil {
return nil, fmt.Errorf("failed to create image reference %q: %w", r, err)
}
// Update image store to reflect the newest state in containerd.
@ -265,12 +269,12 @@ func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) {
// Note that because create and update are not finished in one transaction, there could be race. E.g.
// the image reference is deleted by someone else after create returns already exists, but before update
// happens.
func (c *criService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor) error {
func (c *criService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor, labels map[string]string) error {
img := containerdimages.Image{
Name: name,
Target: desc,
// Add a label to indicate that the image is managed by the cri plugin.
Labels: map[string]string{imageLabelKey: imageLabelValue},
Labels: labels,
}
// TODO(random-liu): Figure out which is the more performant sequence create then update or
// update then create.
@ -278,13 +282,31 @@ func (c *criService) createImageReference(ctx context.Context, name string, desc
if err == nil || !errdefs.IsAlreadyExists(err) {
return err
}
if oldImg.Target.Digest == img.Target.Digest && oldImg.Labels[imageLabelKey] == imageLabelValue {
if oldImg.Target.Digest == img.Target.Digest && oldImg.Labels[crilabels.ImageLabelKey] == labels[crilabels.ImageLabelKey] {
return nil
}
_, err = c.client.ImageService().Update(ctx, img, "target", "labels."+imageLabelKey)
_, err = c.client.ImageService().Update(ctx, img, "target", "labels."+crilabels.ImageLabelKey)
return err
}
// getLabels get image labels to be added on CRI image
func (c *criService) getLabels(ctx context.Context, name string) map[string]string {
labels := map[string]string{crilabels.ImageLabelKey: crilabels.ImageLabelValue}
configSandboxImage := c.config.SandboxImage
// parse sandbox image
sandboxNamedRef, err := distribution.ParseDockerRef(configSandboxImage)
if err != nil {
log.G(ctx).Errorf("failed to parse sandbox image from config %s", sandboxNamedRef)
return nil
}
sandboxRef := sandboxNamedRef.String()
// Adding pinned image label to sandbox image
if sandboxRef == name {
labels[crilabels.PinnedImageLabelKey] = crilabels.PinnedImageLabelValue
}
return labels
}
// updateImage updates image store to reflect the newest state of an image reference
// in containerd. If the reference is not managed by the cri plugin, the function also
// generates necessary metadata for the image and make it managed.
@ -293,7 +315,7 @@ func (c *criService) updateImage(ctx context.Context, r string) error {
if err != nil && !errdefs.IsNotFound(err) {
return fmt.Errorf("get image by reference: %w", err)
}
if err == nil && img.Labels()[imageLabelKey] != imageLabelValue {
if err == nil && img.Labels()[crilabels.ImageLabelKey] != crilabels.ImageLabelValue {
// Make sure the image has the image id as its unique
// identifier that references the image in its lifetime.
configDesc, err := img.Config(ctx)
@ -301,14 +323,15 @@ func (c *criService) updateImage(ctx context.Context, r string) error {
return fmt.Errorf("get image id: %w", err)
}
id := configDesc.Digest.String()
if err := c.createImageReference(ctx, id, img.Target()); err != nil {
labels := c.getLabels(ctx, id)
if err := c.createImageReference(ctx, id, img.Target(), labels); err != nil {
return fmt.Errorf("create image id reference %q: %w", id, err)
}
if err := c.imageStore.Update(ctx, id); err != nil {
return fmt.Errorf("update image store for %q: %w", id, err)
}
// The image id is ready, add the label to mark the image as managed.
if err := c.createImageReference(ctx, r, img.Target()); err != nil {
if err := c.createImageReference(ctx, r, img.Target(), labels); err != nil {
return fmt.Errorf("create managed label: %w", err)
}
}

60
pkg/cri/server/image_pull_test.go
View File

@ -22,11 +22,12 @@ import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
"github.com/containerd/containerd/pkg/cri/labels"
"github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
)
func TestParseAuth(t *testing.T) {
@ -434,3 +435,56 @@ func TestSnapshotterFromPodSandboxConfig(t *testing.T) {
})
}
}
func TestImageGetLabels(t *testing.T) {
criService := newTestCRIService()
tests := []struct {
name string
expectedLabel map[string]string
configSandboxImage string
pullImageName string
}{
{
name: "pinned image labels should get added on sandbox image",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause:3.9",
pullImageName: "k8s.gcr.io/pause:3.9",
},
{
name: "pinned image labels should get added on sandbox image without tag",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause",
pullImageName: "k8s.gcr.io/pause:latest",
},
{
name: "pinned image labels should get added on sandbox image specified with tag and digest both",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause:3.9@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
},
{
name: "pinned image labels should get added on sandbox image specified with digest",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
},
{
name: "pinned image labels should not get added on other image",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue},
configSandboxImage: "k8s.gcr.io/pause:3.9",
pullImageName: "k8s.gcr.io/random:latest",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
criService.config.SandboxImage = tt.configSandboxImage
labels := criService.getLabels(context.Background(), tt.pullImageName)
assert.Equal(t, tt.expectedLabel, labels)
})
}
}

3
pkg/cri/server/image_status.go
View File

@ -63,12 +63,15 @@ func (c *criService) ImageStatus(ctx context.Context, r *runtime.ImageStatusRequ
// toCRIImage converts internal image object to CRI runtime.Image.
func toCRIImage(image imagestore.Image) *runtime.Image {
repoTags, repoDigests := parseImageReferences(image.References)
runtimeImage := &runtime.Image{
Id: image.ID,
RepoTags: repoTags,
RepoDigests: repoDigests,
Size_: uint64(image.Size),
Pinned: image.Pinned,
}
uid, username := getUserFromImage(image.ImageSpec.Config.User)
if uid != nil {
runtimeImage.Uid = &runtime.Int64Value{Value: *uid}

8
pkg/cri/store/image/image.go
View File

@ -23,6 +23,7 @@ import (
"github.com/containerd/containerd"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/pkg/cri/labels"
"github.com/containerd/containerd/pkg/cri/util"
"github.com/containerd/containerd/reference/docker"
@ -45,6 +46,8 @@ type Image struct {
Size int64
// ImageSpec is the oci image structure which describes basic information about the image.
ImageSpec imagespec.Image
// Pinned image to prevent it from garbage collection
Pinned bool
}
// Store stores all images.
@ -131,7 +134,6 @@ func getImage(ctx context.Context, i containerd.Image) (*Image, error) {
if err != nil {
return nil, fmt.Errorf("get image config descriptor: %w", err)
}
id := desc.Digest.String()
spec, err := i.Spec(ctx)
@ -139,13 +141,17 @@ func getImage(ctx context.Context, i containerd.Image) (*Image, error) {
return nil, fmt.Errorf("failed to get OCI image spec: %w", err)
}
pinned := i.Labels()[labels.PinnedImageLabelKey] == labels.PinnedImageLabelValue
return &Image{
ID: id,
References: []string{i.Name()},
ChainID: chainID.String(),
Size: size,
ImageSpec: spec,
Pinned: pinned,
}, nil
}
// Resolve resolves a image reference to image id.