From 3baf5edb8b5b3342c1d729ae857581865cb43b21 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 22 Dec 2023 23:40:21 -0800 Subject: [PATCH] Separate the CRI image config from the main plugin config This change simplifies the CRI plugin dependencies by not requiring the CRI image plugin to depend on any other CRI components. Since other CRI plugins depend on the image plugin, this allows prevents a dependency cycle for CRI configurations on a base plugin. Signed-off-by: Derek McGowan --- cmd/containerd/builtins/cri.go | 1 + .../build_local_containerd_helper_test.go | 13 +- pkg/cri/config/config.go | 97 +++---- pkg/cri/config/config_test.go | 146 ++++++----- pkg/cri/config/config_unix.go | 29 ++- pkg/cri/config/config_windows.go | 25 +- pkg/cri/cri.go | 40 +-- pkg/cri/server/images/image_pull_test.go | 16 +- pkg/cri/server/images/service.go | 155 +----------- pkg/cri/server/images/service_test.go | 20 +- pkg/cri/server/podsandbox/controller.go | 7 +- pkg/cri/server/podsandbox/controller_test.go | 5 - pkg/cri/server/podsandbox/sandbox_run.go | 9 +- pkg/cri/server/service.go | 7 +- pkg/cri/server/test_config.go | 8 - plugins/cri/images/plugin.go | 238 ++++++++++++++++++ 16 files changed, 454 insertions(+), 362 deletions(-) create mode 100644 plugins/cri/images/plugin.go diff --git a/cmd/containerd/builtins/cri.go b/cmd/containerd/builtins/cri.go index 8e8526869..3673889d3 100644 --- a/cmd/containerd/builtins/cri.go +++ b/cmd/containerd/builtins/cri.go @@ -20,4 +20,5 @@ package builtins import ( _ "github.com/containerd/containerd/v2/pkg/cri" + _ "github.com/containerd/containerd/v2/plugins/cri/images" ) diff --git a/integration/build_local_containerd_helper_test.go b/integration/build_local_containerd_helper_test.go index 790399705..6cb14eb5c 100644 --- a/integration/build_local_containerd_helper_test.go +++ b/integration/build_local_containerd_helper_test.go @@ -39,6 +39,7 @@ import ( _ "github.com/containerd/containerd/v2/gc/scheduler" _ "github.com/containerd/containerd/v2/leases/plugin" _ "github.com/containerd/containerd/v2/metadata/plugin" + _ "github.com/containerd/containerd/v2/plugins/cri/images" _ "github.com/containerd/containerd/v2/runtime/v2" _ "github.com/containerd/containerd/v2/runtime/v2/runc/options" _ "github.com/containerd/containerd/v2/services/containers" @@ -53,7 +54,7 @@ import ( _ "github.com/containerd/containerd/v2/services/tasks" _ "github.com/containerd/containerd/v2/services/version" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var ( @@ -72,7 +73,7 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string, tweakInitFn tweakPl // load plugins loadPluginOnce.Do(func() { loadedPlugins, loadedPluginsErr = ctrdsrv.LoadPlugins(ctx, &srvconfig.Config{}) - assert.NoError(t, loadedPluginsErr) + require.NoError(t, loadedPluginsErr) }) // init plugins @@ -104,7 +105,7 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string, tweakInitFn tweakPl // load the plugin specific configuration if it is provided if p.Config != nil { pc, err := config.Decode(ctx, p.URI(), p.Config) - assert.NoError(t, err) + require.NoError(t, err) initContext.Config = pc } @@ -114,10 +115,10 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string, tweakInitFn tweakPl } result := p.Init(initContext) - assert.NoError(t, initialized.Add(result)) + require.NoError(t, initialized.Add(result)) _, err := result.Instance() - assert.NoError(t, err) + require.NoError(t, err) lastInitContext = initContext } @@ -129,7 +130,7 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string, tweakInitFn tweakPl containerd.WithInMemoryServices(lastInitContext), containerd.WithInMemorySandboxControllers(lastInitContext), ) - assert.NoError(t, err) + require.NoError(t, err) return client } diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index 3c8ce8cd5..98fafa4d1 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -67,6 +67,9 @@ const ( ModePodSandbox SandboxControllerMode = "podsandbox" // ModeShim means use whatever Controller implementation provided by shim. ModeShim SandboxControllerMode = "shim" + // DefaultSandboxImage is the default image to use for sandboxes when empty or + // for default configurations. + DefaultSandboxImage = "registry.k8s.io/pause:3.9" ) // Runtime struct to contain the type(ID), engine, and root variables for a default runtime @@ -242,8 +245,12 @@ type ImageDecryption struct { // can be assumed. When platform is not provided, the default platform can // be assumed type ImagePlatform struct { - Platform string - Snapshotter string + Platform string `toml:"platform" json:"platform"` + // Snapshotter setting snapshotter at runtime level instead of making it as a global configuration. + // An example use case is to use devmapper or other snapshotters in Kata containers for performance and security + // while using default snapshotters for operational simplicity. + // See https://github.com/containerd/containerd/issues/6657 for details. + Snapshotter string `toml:"snapshotter" json:"snapshotter"` } type ImageConfig struct { @@ -261,18 +268,21 @@ type ImageConfig struct { DiscardUnpackedLayers bool `toml:"discard_unpacked_layers" json:"discardUnpackedLayers"` // PinnedImages are images which the CRI plugin uses and should not be - // removed by the CRI client. + // removed by the CRI client. The images have a key which can be used + // by other plugins to lookup the current image name. // Image names should be full names including domain and tag // Examples: - // docker.io/library/ubuntu:latest - // images.k8s.io/core/pause:1.55 - PinnedImages []string + // "sandbox": "k8s.gcr.io/pause:3.9" + // "base": "docker.io/library/ubuntu:latest" + // Migrated from: + // (PluginConfig).SandboxImage string `toml:"sandbox_image" json:"sandboxImage"` + PinnedImages map[string]string // RuntimePlatforms is map between the runtime and the image platform to // use for that runtime. When resolving an image for a runtime, this // mapping will be used to select the image for the platform and the // snapshotter for unpacking. - RuntimePlatforms map[string]ImagePlatform + RuntimePlatforms map[string]ImagePlatform `toml:"runtime_platforms" json:"runtimePlatforms"` // Registry contains config related to the registry Registry Registry `toml:"registry" json:"registry"` @@ -305,8 +315,6 @@ type ImageConfig struct { // PluginConfig contains toml config related to CRI plugin, // it is a subset of Config. type PluginConfig struct { - // ImageConfig is the image service configuration - ImageConfig // ContainerdConfig contains config related to containerd ContainerdConfig `toml:"containerd" json:"containerd"` // CniConfig contains config related to cni @@ -327,8 +335,6 @@ type PluginConfig struct { // SelinuxCategoryRange allows the upper bound on the category range to be set. // If not specified or set to 0, defaults to 1024 from the selinux package. SelinuxCategoryRange int `toml:"selinux_category_range" json:"selinuxCategoryRange"` - // SandboxImage is the image used by sandbox container. - SandboxImage string `toml:"sandbox_image" json:"sandboxImage"` // EnableTLSStreaming indicates to enable the TLS streaming support. EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"` // X509KeyPairStreaming is a x509 key pair used for TLS streaming @@ -437,31 +443,9 @@ const ( KeyModelNode = "node" ) -// ValidatePluginConfig validates the given plugin configuration. -func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.Warning, error) { +// ValidateImageConfig validates the given image configuration +func ValidateImageConfig(ctx context.Context, c *ImageConfig) ([]deprecation.Warning, error) { var warnings []deprecation.Warning - if c.ContainerdConfig.Runtimes == nil { - c.ContainerdConfig.Runtimes = make(map[string]Runtime) - } - - // Validation for default_runtime_name - if c.ContainerdConfig.DefaultRuntimeName == "" { - return warnings, errors.New("`default_runtime_name` is empty") - } - if _, ok := c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName]; !ok { - return warnings, fmt.Errorf("no corresponding runtime configured in `containerd.runtimes` for `containerd` `default_runtime_name = \"%s\"", c.ContainerdConfig.DefaultRuntimeName) - } - - for k, r := range c.ContainerdConfig.Runtimes { - if !r.PrivilegedWithoutHostDevices && r.PrivilegedWithoutHostDevicesAllDevicesAllowed { - return warnings, errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled") - } - // If empty, use default podSandbox mode - if len(r.Sandboxer) == 0 { - r.Sandboxer = string(ModePodSandbox) - c.ContainerdConfig.Runtimes[k] = r - } - } useConfigPath := c.Registry.ConfigPath != "" if len(c.Registry.Mirrors) > 0 { @@ -500,13 +484,6 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.W log.G(ctx).Warning("`auths` is deprecated, please use `ImagePullSecrets` instead") } - // Validation for stream_idle_timeout - if c.StreamIdleTimeout != "" { - if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil { - return warnings, fmt.Errorf("invalid stream idle timeout: %w", err) - } - } - // Validation for image_pull_progress_timeout if c.ImagePullProgressTimeout != "" { if _, err := time.ParseDuration(c.ImagePullProgressTimeout); err != nil { @@ -514,6 +491,42 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.W } } + return warnings, nil +} + +// ValidatePluginConfig validates the given plugin configuration. +func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.Warning, error) { + var warnings []deprecation.Warning + if c.ContainerdConfig.Runtimes == nil { + c.ContainerdConfig.Runtimes = make(map[string]Runtime) + } + + // Validation for default_runtime_name + if c.ContainerdConfig.DefaultRuntimeName == "" { + return warnings, errors.New("`default_runtime_name` is empty") + } + if _, ok := c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName]; !ok { + return warnings, fmt.Errorf("no corresponding runtime configured in `containerd.runtimes` for `containerd` `default_runtime_name = \"%s\"", c.ContainerdConfig.DefaultRuntimeName) + } + + for k, r := range c.ContainerdConfig.Runtimes { + if !r.PrivilegedWithoutHostDevices && r.PrivilegedWithoutHostDevicesAllDevicesAllowed { + return warnings, errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled") + } + // If empty, use default podSandbox mode + if len(r.Sandboxer) == 0 { + r.Sandboxer = string(ModePodSandbox) + c.ContainerdConfig.Runtimes[k] = r + } + } + + // Validation for stream_idle_timeout + if c.StreamIdleTimeout != "" { + if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil { + return warnings, fmt.Errorf("invalid stream idle timeout: %w", err) + } + } + // Validation for drain_exec_sync_io_timeout if c.DrainExecSyncIOTimeout != "" { if _, err := time.ParseDuration(c.DrainExecSyncIOTimeout); err != nil { diff --git a/pkg/cri/config/config_test.go b/pkg/cri/config/config_test.go index d4233a4b6..a52b87df5 100644 --- a/pkg/cri/config/config_test.go +++ b/pkg/cri/config/config_test.go @@ -28,10 +28,13 @@ import ( func TestValidateConfig(t *testing.T) { for desc, test := range map[string]struct { - config *PluginConfig - expectedErr string - expected *PluginConfig - warnings []deprecation.Warning + config *PluginConfig + expectedErr string + expected *PluginConfig + imageConfig *ImageConfig + imageExpectedErr string + imageExpected *ImageConfig + warnings []deprecation.Warning }{ "no default_runtime_name": { config: &PluginConfig{}, @@ -54,13 +57,6 @@ func TestValidateConfig(t *testing.T) { RuntimeDefault: {}, }, }, - ImageConfig: ImageConfig{ - Registry: Registry{ - Auths: map[string]AuthConfig{ - "https://gcr.io": {Username: "test"}, - }, - }, - }, }, expected: &PluginConfig{ ContainerdConfig: ContainerdConfig{ @@ -71,18 +67,25 @@ func TestValidateConfig(t *testing.T) { }, }, }, - ImageConfig: ImageConfig{ - Registry: Registry{ - Configs: map[string]RegistryConfig{ - "gcr.io": { - Auth: &AuthConfig{ - Username: "test", - }, + }, + imageConfig: &ImageConfig{ + Registry: Registry{ + Auths: map[string]AuthConfig{ + "https://gcr.io": {Username: "test"}, + }, + }, + }, + imageExpected: &ImageConfig{ + Registry: Registry{ + Configs: map[string]RegistryConfig{ + "gcr.io": { + Auth: &AuthConfig{ + Username: "test", }, }, - Auths: map[string]AuthConfig{ - "https://gcr.io": {Username: "test"}, - }, + }, + Auths: map[string]AuthConfig{ + "https://gcr.io": {Username: "test"}, }, }, }, @@ -103,25 +106,15 @@ func TestValidateConfig(t *testing.T) { expectedErr: "invalid stream idle timeout", }, "conflicting mirror registry config": { - config: &PluginConfig{ - ContainerdConfig: ContainerdConfig{ - DefaultRuntimeName: RuntimeDefault, - Runtimes: map[string]Runtime{ - RuntimeDefault: { - Type: "default", - }, - }, - }, - ImageConfig: ImageConfig{ - Registry: Registry{ - ConfigPath: "/etc/containerd/conf.d", - Mirrors: map[string]Mirror{ - "something.io": {}, - }, + imageConfig: &ImageConfig{ + Registry: Registry{ + ConfigPath: "/etc/containerd/conf.d", + Mirrors: map[string]Mirror{ + "something.io": {}, }, }, }, - expectedErr: "`mirrors` cannot be set when `config_path` is provided", + imageExpectedErr: "`mirrors` cannot be set when `config_path` is provided", }, "deprecated mirrors": { config: &PluginConfig{ @@ -131,11 +124,11 @@ func TestValidateConfig(t *testing.T) { RuntimeDefault: {}, }, }, - ImageConfig: ImageConfig{ - Registry: Registry{ - Mirrors: map[string]Mirror{ - "example.com": {}, - }, + }, + imageConfig: &ImageConfig{ + Registry: Registry{ + Mirrors: map[string]Mirror{ + "example.com": {}, }, }, }, @@ -148,11 +141,11 @@ func TestValidateConfig(t *testing.T) { }, }, }, - ImageConfig: ImageConfig{ - Registry: Registry{ - Mirrors: map[string]Mirror{ - "example.com": {}, - }, + }, + imageExpected: &ImageConfig{ + Registry: Registry{ + Mirrors: map[string]Mirror{ + "example.com": {}, }, }, }, @@ -166,13 +159,13 @@ func TestValidateConfig(t *testing.T) { RuntimeDefault: {}, }, }, - ImageConfig: ImageConfig{ - Registry: Registry{ - Configs: map[string]RegistryConfig{ - "gcr.io": { - Auth: &AuthConfig{ - Username: "test", - }, + }, + imageConfig: &ImageConfig{ + Registry: Registry{ + Configs: map[string]RegistryConfig{ + "gcr.io": { + Auth: &AuthConfig{ + Username: "test", }, }, }, @@ -187,13 +180,13 @@ func TestValidateConfig(t *testing.T) { }, }, }, - ImageConfig: ImageConfig{ - Registry: Registry{ - Configs: map[string]RegistryConfig{ - "gcr.io": { - Auth: &AuthConfig{ - Username: "test", - }, + }, + imageExpected: &ImageConfig{ + Registry: Registry{ + Configs: map[string]RegistryConfig{ + "gcr.io": { + Auth: &AuthConfig{ + Username: "test", }, }, }, @@ -232,17 +225,32 @@ func TestValidateConfig(t *testing.T) { }, } { t.Run(desc, func(t *testing.T) { - w, err := ValidatePluginConfig(context.Background(), test.config) - if test.expectedErr != "" { - assert.Contains(t, err.Error(), test.expectedErr) - } else { - assert.NoError(t, err) - assert.Equal(t, test.expected, test.config) + var warnings []deprecation.Warning + if test.config != nil { + w, err := ValidatePluginConfig(context.Background(), test.config) + if test.expectedErr != "" { + assert.Contains(t, err.Error(), test.expectedErr) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, test.config) + } + warnings = append(warnings, w...) } + if test.imageConfig != nil { + w, err := ValidateImageConfig(context.Background(), test.imageConfig) + if test.imageExpectedErr != "" { + assert.Contains(t, err.Error(), test.imageExpectedErr) + } else { + assert.NoError(t, err) + assert.Equal(t, test.imageExpected, test.imageConfig) + } + warnings = append(warnings, w...) + } + if len(test.warnings) > 0 { - assert.ElementsMatch(t, test.warnings, w) + assert.ElementsMatch(t, test.warnings, warnings) } else { - assert.Len(t, w, 0) + assert.Len(t, warnings, 0) } }) } diff --git a/pkg/cri/config/config_unix.go b/pkg/cri/config/config_unix.go index 7092f044d..269b0db77 100644 --- a/pkg/cri/config/config_unix.go +++ b/pkg/cri/config/config_unix.go @@ -24,6 +24,23 @@ import ( "k8s.io/kubelet/pkg/cri/streaming" ) +func DefaultImageConfig() ImageConfig { + return ImageConfig{ + Snapshotter: containerd.DefaultSnapshotter, + DisableSnapshotAnnotations: true, + MaxConcurrentDownloads: 3, + ImageDecryption: ImageDecryption{ + KeyModel: KeyModelNode, + }, + PinnedImages: map[string]string{ + "sandbox": DefaultSandboxImage, + }, + ImagePullProgressTimeout: defaultImagePullProgressTimeoutDuration.String(), + ImagePullWithSyncFs: false, + StatsCollectPeriod: 10, + } +} + // DefaultConfig returns default configurations of cri plugin. func DefaultConfig() PluginConfig { defaultRuncV2Opts := ` @@ -62,17 +79,6 @@ func DefaultConfig() PluginConfig { NetworkPluginSetupSerially: false, NetworkPluginConfTemplate: "", }, - ImageConfig: ImageConfig{ - Snapshotter: containerd.DefaultSnapshotter, - DisableSnapshotAnnotations: true, - MaxConcurrentDownloads: 3, - ImageDecryption: ImageDecryption{ - KeyModel: KeyModelNode, - }, - ImagePullProgressTimeout: defaultImagePullProgressTimeoutDuration.String(), - ImagePullWithSyncFs: false, - StatsCollectPeriod: 10, - }, ContainerdConfig: ContainerdConfig{ DefaultRuntimeName: "runc", Runtimes: map[string]Runtime{ @@ -94,7 +100,6 @@ func DefaultConfig() PluginConfig { TLSKeyFile: "", TLSCertFile: "", }, - SandboxImage: "registry.k8s.io/pause:3.9", MaxContainerLogLineSize: 16 * 1024, DisableProcMount: false, TolerateMissingHugetlbController: true, diff --git a/pkg/cri/config/config_windows.go b/pkg/cri/config/config_windows.go index f009c9a14..f571a9405 100644 --- a/pkg/cri/config/config_windows.go +++ b/pkg/cri/config/config_windows.go @@ -24,6 +24,21 @@ import ( "k8s.io/kubelet/pkg/cri/streaming" ) +func DefaultImageConfig() ImageConfig { + return ImageConfig{ + Snapshotter: containerd.DefaultSnapshotter, + StatsCollectPeriod: 10, + MaxConcurrentDownloads: 3, + ImageDecryption: ImageDecryption{ + KeyModel: KeyModelNode, + }, + PinnedImages: map[string]string{ + "sandbox": DefaultSandboxImage, + }, + ImagePullProgressTimeout: defaultImagePullProgressTimeoutDuration.String(), + } +} + // DefaultConfig returns default configurations of cri plugin. func DefaultConfig() PluginConfig { return PluginConfig{ @@ -34,15 +49,6 @@ func DefaultConfig() PluginConfig { NetworkPluginSetupSerially: false, NetworkPluginConfTemplate: "", }, - ImageConfig: ImageConfig{ - Snapshotter: containerd.DefaultSnapshotter, - StatsCollectPeriod: 10, - MaxConcurrentDownloads: 3, - ImageDecryption: ImageDecryption{ - KeyModel: KeyModelNode, - }, - ImagePullProgressTimeout: defaultImagePullProgressTimeoutDuration.String(), - }, ContainerdConfig: ContainerdConfig{ DefaultRuntimeName: "runhcs-wcow-process", Runtimes: map[string]Runtime{ @@ -81,7 +87,6 @@ func DefaultConfig() PluginConfig { TLSKeyFile: "", TLSCertFile: "", }, - SandboxImage: "registry.k8s.io/pause:3.9", MaxContainerLogLineSize: 16 * 1024, IgnoreImageDefinedVolumes: false, // TODO(windows): Add platform specific config, so that most common defaults can be shared. diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index 2399ac1e2..95772a544 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -31,7 +31,6 @@ import ( "github.com/containerd/containerd/v2/pkg/cri/nri" "github.com/containerd/containerd/v2/pkg/cri/server" "github.com/containerd/containerd/v2/pkg/cri/server/base" - "github.com/containerd/containerd/v2/pkg/cri/server/images" nriservice "github.com/containerd/containerd/v2/pkg/nri" "github.com/containerd/containerd/v2/platforms" "github.com/containerd/containerd/v2/plugins" @@ -75,11 +74,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { c := criBase.Config // Get image service. - criImagePlugin, err := ic.GetByID(plugins.CRIImagePlugin, "cri-image-service") + criImagePlugin, err := ic.GetSingle(plugins.CRIImagePlugin) if err != nil { return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err) } - imageService := criImagePlugin.(*images.CRIImageService) log.G(ctx).Info("Connect containerd service") client, err := containerd.New( @@ -93,48 +91,20 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { return nil, fmt.Errorf("failed to create containerd client: %w", err) } - // TODO: Update this logic to use runtime snapshotter - if client.SnapshotService(c.ImageConfig.Snapshotter) == nil { - return nil, fmt.Errorf("failed to find snapshotter %q", c.ImageConfig.Snapshotter) - } - // TODO(dmcgowan): Get the full list directly from configured plugins sbControllers := map[string]sandbox.Controller{ string(criconfig.ModePodSandbox): client.SandboxController(string(criconfig.ModePodSandbox)), string(criconfig.ModeShim): client.SandboxController(string(criconfig.ModeShim)), } - //imageFSPaths := map[string]string{} - //for _, ociRuntime := range c.ContainerdConfig.Runtimes { - // // Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.` - // snapshotter := ociRuntime.Snapshotter - // if snapshotter != "" { - // imageFSPaths[snapshotter] = filepath.Join(c.ContainerdRootDir, plugins.SnapshotPlugin.String()+"."+snapshotter) - // log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) - // } - // if _, ok := sbControllers[ociRuntime.Sandboxer]; !ok { - // sbControllers[ociRuntime.Sandboxer] = client.SandboxController(ociRuntime.Sandboxer) - // } - //} - //snapshotter := c.ContainerdConfig.Snapshotter - //imageFSPaths[snapshotter] = filepath.Join(c.ContainerdRootDir, plugins.SnapshotPlugin.String()+"."+snapshotter) - //log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) - - // TODO: expose this as a separate containerd plugin. - //imageService, err := images.NewService(c, imageFSPaths, client) - //if err != nil { - // return nil, fmt.Errorf("unable to create CRI image service: %w", err) - //} - options := &server.CRIServiceOptions{ - ImageService: imageService, + ImageService: criImagePlugin.(server.ImageService), NRI: getNRIAPI(ic), - ImageFSPaths: imageService.ImageFSPaths(), Client: client, SandboxControllers: sbControllers, BaseOCISpecs: criBase.BaseOCISpecs, } - is := images.NewGRPCService(imageService) + is := criImagePlugin.(imageService).GRPCService() s, rs, err := server.NewCRIService(criBase.Config, options) if err != nil { @@ -164,6 +134,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { return criGRPCServerWithTCP{service}, nil } +type imageService interface { + GRPCService() runtime.ImageServiceServer +} + type initializer interface { IsInitialized() bool } diff --git a/pkg/cri/server/images/image_pull_test.go b/pkg/cri/server/images/image_pull_test.go index c86c04357..8ffece4e5 100644 --- a/pkg/cri/server/images/image_pull_test.go +++ b/pkg/cri/server/images/image_pull_test.go @@ -492,27 +492,27 @@ func TestImageGetLabels(t *testing.T) { tests := []struct { name string expectedLabel map[string]string - pinnedImages []string + pinnedImages map[string]string pullImageName string }{ { name: "pinned image labels should get added on sandbox image", expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue}, - pinnedImages: []string{"k8s.gcr.io/pause:3.9"}, + pinnedImages: map[string]string{"sandbox": "k8s.gcr.io/pause:3.9"}, pullImageName: "k8s.gcr.io/pause:3.9", }, { name: "pinned image labels should get added on sandbox image without tag", expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue}, - pinnedImages: []string{"k8s.gcr.io/pause", "k8s.gcr.io/pause:latest"}, + pinnedImages: map[string]string{"sandboxnotag": "k8s.gcr.io/pause", "sandbox": "k8s.gcr.io/pause:latest"}, pullImageName: "k8s.gcr.io/pause:latest", }, { name: "pinned image labels should get added on sandbox image specified with tag and digest both", expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue}, - pinnedImages: []string{ - "k8s.gcr.io/pause:3.9@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2", - "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2", + pinnedImages: map[string]string{ + "sandboxtagdigest": "k8s.gcr.io/pause:3.9@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2", + "sandbox": "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2", }, pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2", }, @@ -520,14 +520,14 @@ func TestImageGetLabels(t *testing.T) { { name: "pinned image labels should get added on sandbox image specified with digest", expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue, labels.PinnedImageLabelKey: labels.PinnedImageLabelValue}, - pinnedImages: []string{"k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2"}, + pinnedImages: map[string]string{"sandbox": "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2"}, pullImageName: "k8s.gcr.io/pause@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2", }, { name: "pinned image labels should not get added on other image", expectedLabel: map[string]string{labels.ImageLabelKey: labels.ImageLabelValue}, - pinnedImages: []string{"k8s.gcr.io/pause:3.9"}, + pinnedImages: map[string]string{"sandbox": "k8s.gcr.io/pause:3.9"}, pullImageName: "k8s.gcr.io/random:latest", }, } diff --git a/pkg/cri/server/images/service.go b/pkg/cri/server/images/service.go index 883858e8c..0fc170c0d 100644 --- a/pkg/cri/server/images/service.go +++ b/pkg/cri/server/images/service.go @@ -18,150 +18,25 @@ package images import ( "context" - "fmt" - "path/filepath" "time" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/events" "github.com/containerd/containerd/v2/images" - "github.com/containerd/containerd/v2/metadata" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" - "github.com/containerd/containerd/v2/pkg/cri/constants" - "github.com/containerd/containerd/v2/pkg/cri/server/base" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot" "github.com/containerd/containerd/v2/pkg/kmutex" "github.com/containerd/containerd/v2/platforms" - "github.com/containerd/containerd/v2/plugins" "github.com/containerd/containerd/v2/snapshots" "github.com/containerd/log" - "github.com/containerd/plugin" - "github.com/containerd/plugin/registry" docker "github.com/distribution/reference" imagedigest "github.com/opencontainers/go-digest" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) -func init() { - registry.Register(&plugin.Registration{ - Type: plugins.CRIImagePlugin, - ID: "cri-image-service", - Requires: []plugin.Type{ - plugins.LeasePlugin, - plugins.EventPlugin, - plugins.MetadataPlugin, - plugins.SandboxStorePlugin, - plugins.InternalPlugin, - plugins.ServicePlugin, - plugins.SnapshotPlugin, - }, - InitFn: func(ic *plugin.InitContext) (interface{}, error) { - // Get base CRI dependencies. - criPlugin, err := ic.GetByID(plugins.InternalPlugin, "cri") - if err != nil { - return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) - } - // TODO: Move this to migration directly - c := criPlugin.(*base.CRIBase).Config - - m, err := ic.GetSingle(plugins.MetadataPlugin) - if err != nil { - return nil, err - } - mdb := m.(*metadata.DB) - - ep, err := ic.GetSingle(plugins.EventPlugin) - if err != nil { - return nil, err - } - - options := &CRIImageServiceOptions{ - Content: mdb.ContentStore(), - Images: metadata.NewImageStore(mdb), - RuntimePlatforms: map[string]ImagePlatform{}, - Snapshotters: map[string]snapshots.Snapshotter{}, - ImageFSPaths: map[string]string{}, - Publisher: ep.(events.Publisher), - } - - options.Client, err = containerd.New( - "", - containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), - containerd.WithDefaultPlatform(platforms.Default()), - containerd.WithInMemoryServices(ic), - ) - if err != nil { - return nil, fmt.Errorf("unable to init client for cri image service: %w", err) - } - - allSnapshotters := mdb.Snapshotters() - defaultSnapshotter := c.ImageConfig.Snapshotter - if s, ok := allSnapshotters[defaultSnapshotter]; ok { - options.Snapshotters[defaultSnapshotter] = s - } else { - return nil, fmt.Errorf("failed to find snapshotter %q", defaultSnapshotter) - } - var snapshotRoot string - if plugin := ic.Plugins().Get(plugins.SnapshotPlugin, defaultSnapshotter); plugin != nil { - snapshotRoot = plugin.Meta.Exports["root"] - } - if snapshotRoot == "" { - // Try a root in the same parent as this plugin - snapshotRoot = filepath.Join(filepath.Dir(ic.Properties[plugins.PropertyRootDir]), plugins.SnapshotPlugin.String()+"."+defaultSnapshotter) - } - options.ImageFSPaths[defaultSnapshotter] = snapshotRoot - log.L.Infof("Get image filesystem path %q for snapshotter %q", snapshotRoot, defaultSnapshotter) - - for runtimeName, rp := range c.ImageConfig.RuntimePlatforms { - // Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.` - snapshotter := rp.Snapshotter - if snapshotter == "" { - snapshotter = defaultSnapshotter - } else if _, ok := options.ImageFSPaths[snapshotter]; !ok { - if s, ok := options.Snapshotters[defaultSnapshotter]; ok { - options.Snapshotters[defaultSnapshotter] = s - } else { - return nil, fmt.Errorf("failed to find snapshotter %q", defaultSnapshotter) - } - var snapshotRoot string - if plugin := ic.Plugins().Get(plugins.SnapshotPlugin, snapshotter); plugin != nil { - snapshotRoot = plugin.Meta.Exports["root"] - } - if snapshotRoot == "" { - // Try a root in the same parent as this plugin - snapshotRoot = filepath.Join(filepath.Dir(ic.Properties[plugins.PropertyRootDir]), plugins.SnapshotPlugin.String()+"."+snapshotter) - } - - options.ImageFSPaths[defaultSnapshotter] = snapshotRoot - log.L.Infof("Get image filesystem path %q for snapshotter %q", options.ImageFSPaths[snapshotter], snapshotter) - } - platform := platforms.DefaultSpec() - if rp.Platform != "" { - p, err := platforms.Parse(rp.Platform) - if err != nil { - return nil, fmt.Errorf("unable to parse platform %q: %w", rp.Platform, err) - } - platform = p - } - options.RuntimePlatforms[runtimeName] = ImagePlatform{ - Snapshotter: snapshotter, - Platform: platform, - } - } - - service, err := NewService(c.ImageConfig, options) - if err != nil { - return nil, fmt.Errorf("failed to create image service: %w", err) - } - - return service, nil - }, - }) -} - type imageClient interface { ListImages(context.Context, ...string) ([]containerd.Image, error) GetImage(context.Context, string) (containerd.Image, error) @@ -174,19 +49,7 @@ type ImagePlatform struct { } type CRIImageService struct { - // config contains all configurations. - // TODO: Migrate configs from cri type once moved to its own plugin - // - snapshotter - // - runtime snapshotter - // - Discard unpack layers - // - Disable snapshot annotations - // - Max concurrent downloads (moved to transfer service) - // - Pull progress timeout - // - Registry headers (moved to transfer service) - // - mirror (moved to transfer service) - // - image decryption (moved to transfer service) - // - default runtime - // - stats collection interval (only used to startup snapshot sync) + // config contains all image configurations. config criconfig.ImageConfig // images is the lower level image store used for raw storage, // no event publishing should currently be assumed @@ -263,11 +126,6 @@ func NewService(config criconfig.ImageConfig, options *CRIImageServiceOptions) ( return &svc, nil } -// NewGRPCService creates a new CRI Image Service grpc server. -func NewGRPCService(imageService *CRIImageService) runtime.ImageServiceServer { - return &GRPCCRIImageService{imageService} -} - // LocalResolve resolves image reference locally and returns corresponding image metadata. It // returns errdefs.ErrNotFound if the reference doesn't exist. func (c *CRIImageService) LocalResolve(refOrID string) (imagestore.Image, error) { @@ -327,3 +185,14 @@ func (c *CRIImageService) GetSnapshot(key, snapshotter string) (snapshotstore.Sn func (c *CRIImageService) ImageFSPaths() map[string]string { return c.imageFSPaths } + +// PinnedImage is used to lookup a pinned image by name. +// Most often used to get the "sandbox" image. +func (c *CRIImageService) PinnedImage(name string) string { + return c.config.PinnedImages[name] +} + +// GRPCService returns a new CRI Image Service grpc server. +func (c *CRIImageService) GRPCService() runtime.ImageServiceServer { + return &GRPCCRIImageService{c} +} diff --git a/pkg/cri/server/images/service_test.go b/pkg/cri/server/images/service_test.go index 5ad88f225..e402f1a74 100644 --- a/pkg/cri/server/images/service_test.go +++ b/pkg/cri/server/images/service_test.go @@ -30,8 +30,6 @@ import ( const ( testImageFSPath = "/test/image/fs/path" - testRootDir = "/test/root" - testStateDir = "/test/state" // Use an image id as test sandbox image to avoid image name resolve. // TODO(random-liu): Change this to image name after we have complete image // management unit test framework. @@ -41,7 +39,7 @@ const ( // newTestCRIService creates a fake criService for test. func newTestCRIService() (*CRIImageService, *GRPCCRIImageService) { service := &CRIImageService{ - config: testConfig.ImageConfig, + config: testImageConfig, runtimePlatforms: map[string]ImagePlatform{}, imageFSPaths: map[string]string{"overlayfs": testImageFSPath}, imageStore: imagestore.NewStore(nil, nil, platforms.Default()), @@ -51,15 +49,9 @@ func newTestCRIService() (*CRIImageService, *GRPCCRIImageService) { return service, &GRPCCRIImageService{service} } -var testConfig = criconfig.Config{ - RootDir: testRootDir, - StateDir: testStateDir, - PluginConfig: criconfig.PluginConfig{ - ImageConfig: criconfig.ImageConfig{ - PinnedImages: []string{testSandboxImage}, - }, - SandboxImage: testSandboxImage, - TolerateMissingHugetlbController: true, +var testImageConfig = criconfig.ImageConfig{ + PinnedImages: map[string]string{ + "sandbox": testSandboxImage, }, } @@ -119,7 +111,7 @@ func TestRuntimeSnapshotter(t *testing.T) { { desc: "should return default snapshotter when runtime.Snapshotter is not set", runtime: defaultRuntime, - expectSnapshotter: criconfig.DefaultConfig().Snapshotter, + expectSnapshotter: criconfig.DefaultImageConfig().Snapshotter, }, { desc: "should return overridden snapshotter when runtime.Snapshotter is set", @@ -130,7 +122,7 @@ func TestRuntimeSnapshotter(t *testing.T) { test := test t.Run(test.desc, func(t *testing.T) { cri, _ := newTestCRIService() - cri.config = criconfig.DefaultConfig().ImageConfig + cri.config = criconfig.DefaultImageConfig() assert.Equal(t, test.expectSnapshotter, cri.RuntimeSnapshotter(context.Background(), test.runtime)) }) } diff --git a/pkg/cri/server/podsandbox/controller.go b/pkg/cri/server/podsandbox/controller.go index cb36b4d8a..240d6f550 100644 --- a/pkg/cri/server/podsandbox/controller.go +++ b/pkg/cri/server/podsandbox/controller.go @@ -33,7 +33,6 @@ import ( criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/constants" "github.com/containerd/containerd/v2/pkg/cri/server/base" - "github.com/containerd/containerd/v2/pkg/cri/server/images" "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox/types" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" @@ -75,18 +74,17 @@ func init() { criBase := criBasePlugin.(*base.CRIBase) // Get image service. - criImagePlugin, err := ic.GetByID(plugins.CRIImagePlugin, "cri-image-service") + criImagePlugin, err := ic.GetSingle(plugins.CRIImagePlugin) if err != nil { return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err) } - imageService := criImagePlugin.(*images.CRIImageService) c := Controller{ client: client, config: criBase.Config, os: osinterface.RealOS{}, baseOCISpecs: criBase.BaseOCISpecs, - imageService: imageService, + imageService: criImagePlugin.(ImageService), store: NewStore(), } return &c, nil @@ -107,6 +105,7 @@ type ImageService interface { GetImage(id string) (imagestore.Image, error) PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig) (string, error) RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string + PinnedImage(string) string } type Controller struct { diff --git a/pkg/cri/server/podsandbox/controller_test.go b/pkg/cri/server/podsandbox/controller_test.go index f8609119f..e71edf580 100644 --- a/pkg/cri/server/podsandbox/controller_test.go +++ b/pkg/cri/server/podsandbox/controller_test.go @@ -33,17 +33,12 @@ import ( const ( testRootDir = "/test/root" testStateDir = "/test/state" - // Use an image id as test sandbox image to avoid image name resolve. - // TODO(random-liu): Change this to image name after we have complete image - // management unit test framework. - testSandboxImage = "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113798" // #nosec G101 ) var testConfig = criconfig.Config{ RootDir: testRootDir, StateDir: testStateDir, PluginConfig: criconfig.PluginConfig{ - SandboxImage: testSandboxImage, TolerateMissingHugetlbController: true, }, } diff --git a/pkg/cri/server/podsandbox/sandbox_run.go b/pkg/cri/server/podsandbox/sandbox_run.go index ec9effb0c..559cc771a 100644 --- a/pkg/cri/server/podsandbox/sandbox_run.go +++ b/pkg/cri/server/podsandbox/sandbox_run.go @@ -32,6 +32,7 @@ import ( containerdio "github.com/containerd/containerd/v2/cio" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/errdefs" + criconfig "github.com/containerd/containerd/v2/pkg/cri/config" crilabels "github.com/containerd/containerd/v2/pkg/cri/labels" customopts "github.com/containerd/containerd/v2/pkg/cri/opts" "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox/types" @@ -74,10 +75,14 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll labels = map[string]string{} ) + sandboxImage := c.imageService.PinnedImage("sandbox") + if sandboxImage == "" { + sandboxImage = criconfig.DefaultSandboxImage + } // Ensure sandbox container image snapshot. - image, err := c.ensureImageExists(ctx, c.config.SandboxImage, config) + image, err := c.ensureImageExists(ctx, sandboxImage, config) if err != nil { - return cin, fmt.Errorf("failed to get sandbox image %q: %w", c.config.SandboxImage, err) + return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err) } containerdImage, err := c.toContainerdImage(ctx, *image) diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index d32654b36..32c322cf7 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -135,11 +135,6 @@ type CRIServiceOptions struct { // SandboxControllers is a map of all the loaded sandbox controllers SandboxControllers map[string]sandbox.Controller - // ImageFSPath is the filesystem path for images - // - // TODO: Move this to cached snapshot metadata - ImageFSPaths map[string]string - // BaseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec` BaseOCISpecs map[string]*oci.Spec @@ -159,7 +154,7 @@ func NewCRIService(config criconfig.Config, options *CRIServiceOptions) (CRIServ ImageService: options.ImageService, config: config, client: options.Client, - imageFSPaths: options.ImageFSPaths, + imageFSPaths: options.ImageService.ImageFSPaths(), os: osinterface.RealOS{}, baseOCISpecs: options.BaseOCISpecs, sandboxStore: sandboxstore.NewStore(labels), diff --git a/pkg/cri/server/test_config.go b/pkg/cri/server/test_config.go index b86e72e4d..a0ec785ff 100644 --- a/pkg/cri/server/test_config.go +++ b/pkg/cri/server/test_config.go @@ -21,21 +21,13 @@ import criconfig "github.com/containerd/containerd/v2/pkg/cri/config" const ( testRootDir = "/test/root" testStateDir = "/test/state" - // Use an image id as test sandbox image to avoid image name resolve. - // TODO(random-liu): Change this to image name after we have complete image - // management unit test framework. - testSandboxImage = "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113798" // #nosec G101 ) var testConfig = criconfig.Config{ RootDir: testRootDir, StateDir: testStateDir, PluginConfig: criconfig.PluginConfig{ - SandboxImage: testSandboxImage, TolerateMissingHugetlbController: true, - ImageConfig: criconfig.ImageConfig{ - Snapshotter: "overlayfs", - }, ContainerdConfig: criconfig.ContainerdConfig{ DefaultRuntimeName: "runc", Runtimes: map[string]criconfig.Runtime{ diff --git a/plugins/cri/images/plugin.go b/plugins/cri/images/plugin.go new file mode 100644 index 000000000..b771268b1 --- /dev/null +++ b/plugins/cri/images/plugin.go @@ -0,0 +1,238 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package images + +import ( + "context" + "fmt" + "path/filepath" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/events" + "github.com/containerd/containerd/v2/metadata" + criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/constants" + "github.com/containerd/containerd/v2/pkg/cri/server/images" + "github.com/containerd/containerd/v2/platforms" + "github.com/containerd/containerd/v2/plugins" + srvconfig "github.com/containerd/containerd/v2/services/server/config" + "github.com/containerd/containerd/v2/snapshots" + "github.com/containerd/log" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" +) + +func init() { + config := criconfig.DefaultImageConfig() + + registry.Register(&plugin.Registration{ + Type: plugins.CRIImagePlugin, + ID: "local", + Config: &config, + Requires: []plugin.Type{ + plugins.LeasePlugin, + plugins.EventPlugin, + plugins.MetadataPlugin, + plugins.SandboxStorePlugin, + plugins.InternalPlugin, // For config migration ordering + plugins.ServicePlugin, // For client + plugins.SnapshotPlugin, // For root directory properties + }, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + m, err := ic.GetSingle(plugins.MetadataPlugin) + if err != nil { + return nil, err + } + mdb := m.(*metadata.DB) + + ep, err := ic.GetSingle(plugins.EventPlugin) + if err != nil { + return nil, err + } + + options := &images.CRIImageServiceOptions{ + Content: mdb.ContentStore(), + Images: metadata.NewImageStore(mdb), + RuntimePlatforms: map[string]images.ImagePlatform{}, + Snapshotters: map[string]snapshots.Snapshotter{}, + ImageFSPaths: map[string]string{}, + Publisher: ep.(events.Publisher), + } + + options.Client, err = containerd.New( + "", + containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), + containerd.WithDefaultPlatform(platforms.Default()), + containerd.WithInMemoryServices(ic), + ) + if err != nil { + return nil, fmt.Errorf("unable to init client for cri image service: %w", err) + } + + allSnapshotters := mdb.Snapshotters() + defaultSnapshotter := config.Snapshotter + if s, ok := allSnapshotters[defaultSnapshotter]; ok { + options.Snapshotters[defaultSnapshotter] = s + } else { + return nil, fmt.Errorf("failed to find snapshotter %q", defaultSnapshotter) + } + var snapshotRoot string + if plugin := ic.Plugins().Get(plugins.SnapshotPlugin, defaultSnapshotter); plugin != nil { + snapshotRoot = plugin.Meta.Exports["root"] + } + if snapshotRoot == "" { + // Try a root in the same parent as this plugin + snapshotRoot = filepath.Join(filepath.Dir(ic.Properties[plugins.PropertyRootDir]), plugins.SnapshotPlugin.String()+"."+defaultSnapshotter) + } + options.ImageFSPaths[defaultSnapshotter] = snapshotRoot + log.L.Infof("Get image filesystem path %q for snapshotter %q", snapshotRoot, defaultSnapshotter) + + for runtimeName, rp := range config.RuntimePlatforms { + snapshotter := rp.Snapshotter + if snapshotter == "" { + snapshotter = defaultSnapshotter + } else if _, ok := options.ImageFSPaths[snapshotter]; !ok { + if s, ok := options.Snapshotters[defaultSnapshotter]; ok { + options.Snapshotters[defaultSnapshotter] = s + } else { + return nil, fmt.Errorf("failed to find snapshotter %q", defaultSnapshotter) + } + var snapshotRoot string + if plugin := ic.Plugins().Get(plugins.SnapshotPlugin, snapshotter); plugin != nil { + snapshotRoot = plugin.Meta.Exports["root"] + } + if snapshotRoot == "" { + // Try a root in the same parent as this plugin + snapshotRoot = filepath.Join(filepath.Dir(ic.Properties[plugins.PropertyRootDir]), plugins.SnapshotPlugin.String()+"."+snapshotter) + } + + options.ImageFSPaths[defaultSnapshotter] = snapshotRoot + log.L.Infof("Get image filesystem path %q for snapshotter %q", options.ImageFSPaths[snapshotter], snapshotter) + } + platform := platforms.DefaultSpec() + if rp.Platform != "" { + p, err := platforms.Parse(rp.Platform) + if err != nil { + return nil, fmt.Errorf("unable to parse platform %q: %w", rp.Platform, err) + } + platform = p + } + options.RuntimePlatforms[runtimeName] = images.ImagePlatform{ + Snapshotter: snapshotter, + Platform: platform, + } + } + + service, err := images.NewService(config, options) + if err != nil { + return nil, fmt.Errorf("failed to create image service: %w", err) + } + + return service, nil + }, + ConfigMigration: configMigration, + }) +} + +func configMigration(ctx context.Context, version int, pluginConfigs map[string]interface{}) error { + if version >= srvconfig.CurrentConfigVersion { + return nil + } + original, ok := pluginConfigs[string(plugins.InternalPlugin)+".cri"] + if !ok { + return nil + } + src := original.(map[string]interface{}) + updated, ok := pluginConfigs[string(plugins.CRIImagePlugin)+".local"] + var dst map[string]interface{} + if ok { + dst = updated.(map[string]interface{}) + } else { + dst = map[string]interface{}{} + } + + migrateConfig(dst, src) + pluginConfigs[string(plugins.CRIImagePlugin)+".local"] = dst + return nil +} +func migrateConfig(dst, src map[string]interface{}) { + containerdConf, ok := src["containerd"] + if !ok { + return + } + containerdConfMap := containerdConf.(map[string]interface{}) + runtimesConf, ok := containerdConfMap["runtimes"] + if !ok { + return + } + + var runtimePlatforms map[string]interface{} + if v, ok := dst["runtime_platform"]; ok { + runtimePlatforms = v.(map[string]interface{}) + } else { + runtimePlatforms = map[string]interface{}{} + } + for runtime, v := range runtimesConf.(map[string]interface{}) { + runtimeConf := v.(map[string]interface{}) + if snapshotter, ok := runtimeConf["snapshot"]; ok && snapshotter != "" { + runtimePlatforms[runtime] = map[string]interface{}{ + "platform": platforms.DefaultStrict(), + "snapshotter": snapshotter, + } + } + } + if len(runtimePlatforms) > 0 { + dst["runtime_platform"] = runtimePlatforms + } + + var pinnedImages map[string]interface{} + if v, ok := dst["pinned_images"]; ok { + pinnedImages = v.(map[string]interface{}) + } else { + pinnedImages = map[string]interface{}{} + } + + if simage, ok := src["sandbox_image"]; ok { + pinnedImages["sandbox"] = simage + } + if len(pinnedImages) > 0 { + dst["pinned_images"] = pinnedImages + } + + for _, key := range []string{ + "snapshotter", + "disable_snapshot_annotations", + "discard_unpacked_layers", + } { + if val, ok := containerdConfMap[key]; ok { + dst[key] = val + } + } + + for _, key := range []string{ + "registry", + "image_decryption", + "max_concurrent_downloads", + "image_pull_progress_timeout", + "image_pull_with_sync_fs", + "stats_collect_period", + } { + if val, ok := src[key]; ok { + dst[key] = val + } + } +}