github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Separate the CRI image config from the main plugin config

Browse Source 
This change simplifies the CRI plugin dependencies by not requiring the
CRI image plugin to depend on any other CRI components. Since other CRI
plugins depend on the image plugin, this allows prevents a dependency
cycle for CRI configurations on a base plugin.

Signed-off-by: Derek McGowan <derek@mcg.dev>
This commit is contained in:
Derek McGowan
2023-12-22 23:40:21 -08:00
parent ad4c9f8a9d
commit 3baf5edb8b
16 changed files with 454 additions and 362 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
pkg/cri/server/images/image_pull_test.go
View File

@@ -492,27 +492,27 @@ func TestImageGetLabels(t *testing.T) {
tests := []struct {
name string
expectedLabel map[string]string
pinnedImages []string
pinnedImages map[string]string
pullImageName string
}{
{
name: "pinned image labels should get added on sandbox image",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
pinnedImages: []string{"k8s.gcr.io/pause:3.9"},
pinnedImages: map[string]string{"sandbox": "k8s.gcr.io/pause:3.9"},
pullImageName: "k8s.gcr.io/pause:3.9",
},
{
name: "pinned image labels should get added on sandbox image without tag",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
pinnedImages: []string{"k8s.gcr.io/pause", "k8s.gcr.io/pause:latest"},
pinnedImages: map[string]string{"sandboxnotag": "k8s.gcr.io/pause", "sandbox": "k8s.gcr.io/pause:latest"},
pullImageName: "k8s.gcr.io/pause:latest",
},
{
name: "pinned image labels should get added on sandbox image specified with tag and digest both",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
pinnedImages: []string{
"k8s.gcr.io/pause:3.9@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
"k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
pinnedImages: map[string]string{
"sandboxtagdigest": "k8s.gcr.io/pause:3.9@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
"sandbox": "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
},
pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
},
@@ -520,14 +520,14 @@ func TestImageGetLabels(t *testing.T) {
{
name: "pinned image labels should get added on sandbox image specified with digest",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue},
pinnedImages: []string{"k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2"},
pinnedImages: map[string]string{"sandbox": "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2"},
pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
},
{
name: "pinned image labels should not get added on other image",
expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue},
pinnedImages: []string{"k8s.gcr.io/pause:3.9"},
pinnedImages: map[string]string{"sandbox": "k8s.gcr.io/pause:3.9"},
pullImageName: "k8s.gcr.io/random:latest",
},
}

155
pkg/cri/server/images/service.go
View File

@@ -18,150 +18,25 @@ package images
import (
"context"
"fmt"
"path/filepath"
"time"
containerd "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/content"
"github.com/containerd/containerd/v2/events"
"github.com/containerd/containerd/v2/images"
"github.com/containerd/containerd/v2/metadata"
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
"github.com/containerd/containerd/v2/pkg/cri/constants"
"github.com/containerd/containerd/v2/pkg/cri/server/base"
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot"
"github.com/containerd/containerd/v2/pkg/kmutex"
"github.com/containerd/containerd/v2/platforms"
"github.com/containerd/containerd/v2/plugins"
"github.com/containerd/containerd/v2/snapshots"
"github.com/containerd/log"
"github.com/containerd/plugin"
"github.com/containerd/plugin/registry"
docker "github.com/distribution/reference"
imagedigest "github.com/opencontainers/go-digest"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
)
func init() {
registry.Register(&plugin.Registration{
Type: plugins.CRIImagePlugin,
ID: "cri-image-service",
Requires: []plugin.Type{
plugins.LeasePlugin,
plugins.EventPlugin,
plugins.MetadataPlugin,
plugins.SandboxStorePlugin,
plugins.InternalPlugin,
plugins.ServicePlugin,
plugins.SnapshotPlugin,
},
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
// Get base CRI dependencies.
criPlugin, err := ic.GetByID(plugins.InternalPlugin, "cri")
if err != nil {
return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err)
}
// TODO: Move this to migration directly
c := criPlugin.(*base.CRIBase).Config
m, err := ic.GetSingle(plugins.MetadataPlugin)
if err != nil {
return nil, err
}
mdb := m.(*metadata.DB)
ep, err := ic.GetSingle(plugins.EventPlugin)
if err != nil {
return nil, err
}
options := &CRIImageServiceOptions{
Content: mdb.ContentStore(),
Images: metadata.NewImageStore(mdb),
RuntimePlatforms: map[string]ImagePlatform{},
Snapshotters: map[string]snapshots.Snapshotter{},
ImageFSPaths: map[string]string{},
Publisher: ep.(events.Publisher),
}
options.Client, err = containerd.New(
"",
containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
containerd.WithDefaultPlatform(platforms.Default()),
containerd.WithInMemoryServices(ic),
)
if err != nil {
return nil, fmt.Errorf("unable to init client for cri image service: %w", err)
}
allSnapshotters := mdb.Snapshotters()
defaultSnapshotter := c.ImageConfig.Snapshotter
if s, ok := allSnapshotters[defaultSnapshotter]; ok {
options.Snapshotters[defaultSnapshotter] = s
} else {
return nil, fmt.Errorf("failed to find snapshotter %q", defaultSnapshotter)
}
var snapshotRoot string
if plugin := ic.Plugins().Get(plugins.SnapshotPlugin, defaultSnapshotter); plugin != nil {
snapshotRoot = plugin.Meta.Exports["root"]
}
if snapshotRoot == "" {
// Try a root in the same parent as this plugin
snapshotRoot = filepath.Join(filepath.Dir(ic.Properties[plugins.PropertyRootDir]), plugins.SnapshotPlugin.String()+"."+defaultSnapshotter)
}
options.ImageFSPaths[defaultSnapshotter] = snapshotRoot
log.L.Infof("Get image filesystem path %q for snapshotter %q", snapshotRoot, defaultSnapshotter)
for runtimeName, rp := range c.ImageConfig.RuntimePlatforms {
// Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.`
snapshotter := rp.Snapshotter
if snapshotter == "" {
snapshotter = defaultSnapshotter
} else if _, ok := options.ImageFSPaths[snapshotter]; !ok {
if s, ok := options.Snapshotters[defaultSnapshotter]; ok {
options.Snapshotters[defaultSnapshotter] = s
} else {
return nil, fmt.Errorf("failed to find snapshotter %q", defaultSnapshotter)
}
var snapshotRoot string
if plugin := ic.Plugins().Get(plugins.SnapshotPlugin, snapshotter); plugin != nil {
snapshotRoot = plugin.Meta.Exports["root"]
}
if snapshotRoot == "" {
// Try a root in the same parent as this plugin
snapshotRoot = filepath.Join(filepath.Dir(ic.Properties[plugins.PropertyRootDir]), plugins.SnapshotPlugin.String()+"."+snapshotter)
}
options.ImageFSPaths[defaultSnapshotter] = snapshotRoot
log.L.Infof("Get image filesystem path %q for snapshotter %q", options.ImageFSPaths[snapshotter], snapshotter)
}
platform := platforms.DefaultSpec()
if rp.Platform != "" {
p, err := platforms.Parse(rp.Platform)
if err != nil {
return nil, fmt.Errorf("unable to parse platform %q: %w", rp.Platform, err)
}
platform = p
}
options.RuntimePlatforms[runtimeName] = ImagePlatform{
Snapshotter: snapshotter,
Platform: platform,
}
}
service, err := NewService(c.ImageConfig, options)
if err != nil {
return nil, fmt.Errorf("failed to create image service: %w", err)
}
return service, nil
},
})
}
type imageClient interface {
ListImages(context.Context, ...string) ([]containerd.Image, error)
GetImage(context.Context, string) (containerd.Image, error)
@@ -174,19 +49,7 @@ type ImagePlatform struct {
}
type CRIImageService struct {
// config contains all configurations.
// TODO: Migrate configs from cri type once moved to its own plugin
// - snapshotter
// - runtime snapshotter
// - Discard unpack layers
// - Disable snapshot annotations
// - Max concurrent downloads (moved to transfer service)
// - Pull progress timeout
// - Registry headers (moved to transfer service)
// - mirror (moved to transfer service)
// - image decryption (moved to transfer service)
// - default runtime
// - stats collection interval (only used to startup snapshot sync)
// config contains all image configurations.
config criconfig.ImageConfig
// images is the lower level image store used for raw storage,
// no event publishing should currently be assumed
@@ -263,11 +126,6 @@ func NewService(config criconfig.ImageConfig, options *CRIImageServiceOptions) (
return &svc, nil
}
// NewGRPCService creates a new CRI Image Service grpc server.
func NewGRPCService(imageService *CRIImageService) runtime.ImageServiceServer {
return &GRPCCRIImageService{imageService}
}
// LocalResolve resolves image reference locally and returns corresponding image metadata. It
// returns errdefs.ErrNotFound if the reference doesn't exist.
func (c *CRIImageService) LocalResolve(refOrID string) (imagestore.Image, error) {
@@ -327,3 +185,14 @@ func (c *CRIImageService) GetSnapshot(key, snapshotter string) (snapshotstore.Sn
func (c *CRIImageService) ImageFSPaths() map[string]string {
return c.imageFSPaths
}
// PinnedImage is used to lookup a pinned image by name.
// Most often used to get the "sandbox" image.
func (c *CRIImageService) PinnedImage(name string) string {
return c.config.PinnedImages[name]
}
// GRPCService returns a new CRI Image Service grpc server.
func (c *CRIImageService) GRPCService() runtime.ImageServiceServer {
return &GRPCCRIImageService{c}
}

20
pkg/cri/server/images/service_test.go
View File

@@ -30,8 +30,6 @@ import (
const (
testImageFSPath = "/test/image/fs/path"
testRootDir = "/test/root"
testStateDir = "/test/state"
// Use an image id as test sandbox image to avoid image name resolve.
// TODO(random-liu): Change this to image name after we have complete image
// management unit test framework.
@@ -41,7 +39,7 @@ const (
// newTestCRIService creates a fake criService for test.
func newTestCRIService() (*CRIImageService, *GRPCCRIImageService) {
service := &CRIImageService{
config: testConfig.ImageConfig,
config: testImageConfig,
runtimePlatforms: map[string]ImagePlatform{},
imageFSPaths: map[string]string{"overlayfs": testImageFSPath},
imageStore: imagestore.NewStore(nil, nil, platforms.Default()),
@@ -51,15 +49,9 @@ func newTestCRIService() (*CRIImageService, *GRPCCRIImageService) {
return service, &GRPCCRIImageService{service}
}
var testConfig = criconfig.Config{
RootDir: testRootDir,
StateDir: testStateDir,
PluginConfig: criconfig.PluginConfig{
ImageConfig: criconfig.ImageConfig{
PinnedImages: []string{testSandboxImage},
},
SandboxImage: testSandboxImage,
TolerateMissingHugetlbController: true,
var testImageConfig = criconfig.ImageConfig{
PinnedImages: map[string]string{
"sandbox": testSandboxImage,
},
}
@@ -119,7 +111,7 @@ func TestRuntimeSnapshotter(t *testing.T) {
{
desc: "should return default snapshotter when runtime.Snapshotter is not set",
runtime: defaultRuntime,
expectSnapshotter: criconfig.DefaultConfig().Snapshotter,
expectSnapshotter: criconfig.DefaultImageConfig().Snapshotter,
},
{
desc: "should return overridden snapshotter when runtime.Snapshotter is set",
@@ -130,7 +122,7 @@ func TestRuntimeSnapshotter(t *testing.T) {
test := test
t.Run(test.desc, func(t *testing.T) {
cri, _ := newTestCRIService()
cri.config = criconfig.DefaultConfig().ImageConfig
cri.config = criconfig.DefaultImageConfig()
assert.Equal(t, test.expectSnapshotter, cri.RuntimeSnapshotter(context.Background(), test.runtime))
})
}

7
pkg/cri/server/podsandbox/controller.go
View File

@@ -33,7 +33,6 @@ import (
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
"github.com/containerd/containerd/v2/pkg/cri/constants"
"github.com/containerd/containerd/v2/pkg/cri/server/base"
"github.com/containerd/containerd/v2/pkg/cri/server/images"
"github.com/containerd/containerd/v2/pkg/cri/server/podsandbox/types"
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util"
@@ -75,18 +74,17 @@ func init() {
criBase := criBasePlugin.(*base.CRIBase)
// Get image service.
criImagePlugin, err := ic.GetByID(plugins.CRIImagePlugin, "cri-image-service")
criImagePlugin, err := ic.GetSingle(plugins.CRIImagePlugin)
if err != nil {
return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err)
}
imageService := criImagePlugin.(*images.CRIImageService)
c := Controller{
client: client,
config: criBase.Config,
os: osinterface.RealOS{},
baseOCISpecs: criBase.BaseOCISpecs,
imageService: imageService,
imageService: criImagePlugin.(ImageService),
store: NewStore(),
}
return &c, nil
@@ -107,6 +105,7 @@ type ImageService interface {
GetImage(id string) (imagestore.Image, error)
PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig) (string, error)
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
PinnedImage(string) string
}
type Controller struct {

5
pkg/cri/server/podsandbox/controller_test.go
View File

@@ -33,17 +33,12 @@ import (
const (
testRootDir = "/test/root"
testStateDir = "/test/state"
// Use an image id as test sandbox image to avoid image name resolve.
// TODO(random-liu): Change this to image name after we have complete image
// management unit test framework.
testSandboxImage = "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113798" // #nosec G101
)
var testConfig = criconfig.Config{
RootDir: testRootDir,
StateDir: testStateDir,
PluginConfig: criconfig.PluginConfig{
SandboxImage: testSandboxImage,
TolerateMissingHugetlbController: true,
},
}

9
pkg/cri/server/podsandbox/sandbox_run.go
View File

@@ -32,6 +32,7 @@ import (
containerdio "github.com/containerd/containerd/v2/cio"
containerd "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/errdefs"
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
crilabels "github.com/containerd/containerd/v2/pkg/cri/labels"
customopts "github.com/containerd/containerd/v2/pkg/cri/opts"
"github.com/containerd/containerd/v2/pkg/cri/server/podsandbox/types"
@@ -74,10 +75,14 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
labels = map[string]string{}
)
sandboxImage := c.imageService.PinnedImage("sandbox")
if sandboxImage == "" {
sandboxImage = criconfig.DefaultSandboxImage
}
// Ensure sandbox container image snapshot.
image, err := c.ensureImageExists(ctx, c.config.SandboxImage, config)
image, err := c.ensureImageExists(ctx, sandboxImage, config)
if err != nil {
return cin, fmt.Errorf("failed to get sandbox image %q: %w", c.config.SandboxImage, err)
return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err)
}
containerdImage, err := c.toContainerdImage(ctx, *image)

7
pkg/cri/server/service.go
View File

@@ -135,11 +135,6 @@ type CRIServiceOptions struct {
// SandboxControllers is a map of all the loaded sandbox controllers
SandboxControllers map[string]sandbox.Controller
// ImageFSPath is the filesystem path for images
//
// TODO: Move this to cached snapshot metadata
ImageFSPaths map[string]string
// BaseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
BaseOCISpecs map[string]*oci.Spec
@@ -159,7 +154,7 @@ func NewCRIService(config criconfig.Config, options *CRIServiceOptions) (CRIServ
ImageService: options.ImageService,
config: config,
client: options.Client,
imageFSPaths: options.ImageFSPaths,
imageFSPaths: options.ImageService.ImageFSPaths(),
os: osinterface.RealOS{},
baseOCISpecs: options.BaseOCISpecs,
sandboxStore: sandboxstore.NewStore(labels),

8
pkg/cri/server/test_config.go
View File

@@ -21,21 +21,13 @@ import criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
const (
testRootDir = "/test/root"
testStateDir = "/test/state"
// Use an image id as test sandbox image to avoid image name resolve.
// TODO(random-liu): Change this to image name after we have complete image
// management unit test framework.
testSandboxImage = "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113798" // #nosec G101
)
var testConfig = criconfig.Config{
RootDir: testRootDir,
StateDir: testStateDir,
PluginConfig: criconfig.PluginConfig{
SandboxImage: testSandboxImage,
TolerateMissingHugetlbController: true,
ImageConfig: criconfig.ImageConfig{
Snapshotter: "overlayfs",
},
ContainerdConfig: criconfig.ContainerdConfig{
DefaultRuntimeName: "runc",
Runtimes: map[string]criconfig.Runtime{