diff --git a/docs/cri/config.md b/docs/cri/config.md index e4d0319cd..df6e95c82 100644 --- a/docs/cri/config.md +++ b/docs/cri/config.md @@ -228,7 +228,8 @@ version = 2 # 'plugins."io.containerd.grpc.v1.cri".containerd' contains config related to containerd [plugins."io.containerd.grpc.v1.cri".containerd] - # snapshotter is the snapshotter used by containerd. + # snapshotter is the default snapshotter used by containerd + # for all runtimes, if not overridden by an experimental runtime's snapshotter config. snapshotter = "overlayfs" # no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc. @@ -329,6 +330,11 @@ version = 2 # config files being loaded from the CNI config directory. cni_max_conf_num = 1 + # snapshotter overrides the global default snapshotter to a runtime specific value. + # Please be aware that overriding the default snapshotter on a runtime basis is currently an experimental feature. + # See https://github.com/containerd/containerd/issues/6657 for context. + snapshotter = "" + # 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to # "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is: # https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 . diff --git a/pkg/cri/annotations/annotations.go b/pkg/cri/annotations/annotations.go index 181eb05c7..97f83095e 100644 --- a/pkg/cri/annotations/annotations.go +++ b/pkg/cri/annotations/annotations.go @@ -70,6 +70,13 @@ const ( // PodAnnotations are the annotations of the pod PodAnnotations = "io.kubernetes.cri.pod-annotations" + // RuntimeHandler an experimental annotation key for getting runtime handler from pod annotations. + // See https://github.com/containerd/containerd/issues/6657 and https://github.com/containerd/containerd/pull/6899 for details. + // The value of this annotation should be the runtime for sandboxes. + // e.g. for [plugins.cri.containerd.runtimes.runc] runtime config, this value should be runc + // TODO: we should deprecate this annotation as soon as kubelet supports passing RuntimeHandler from PullImageRequest + RuntimeHandler = "io.containerd.cri.runtime-handler" + // WindowsHostProcess is used by hcsshim to identify windows pods that are running HostProcesses WindowsHostProcess = "microsoft.com/hostprocess-container" ) diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index 8f1867127..be0a2f15e 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -71,6 +71,11 @@ type Runtime struct { // be loaded from the cni config directory by go-cni. Set the value to 0 to // load all config files (no arbitrary limit). The legacy default value is 1. NetworkPluginMaxConfNum int `toml:"cni_max_conf_num" json:"cniMaxConfNum"` + // Snapshotter setting snapshotter at runtime level instead of making it as a global configuration. + // An example use case is to use devmapper or other snapshotters in Kata containers for performance and security + // while using default snapshotters for operational simplicity. + // See https://github.com/containerd/containerd/issues/6657 for details. + Snapshotter string `toml:"snapshotter" json:"snapshotter"` } // ContainerdConfig contains toml config related to containerd diff --git a/pkg/cri/server/container_create.go b/pkg/cri/server/container_create.go index 0978951bf..e185ebe1c 100644 --- a/pkg/cri/server/container_create.go +++ b/pkg/cri/server/container_create.go @@ -23,11 +23,6 @@ import ( "path/filepath" "time" - "github.com/containerd/containerd" - "github.com/containerd/containerd/containers" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/oci" - "github.com/containerd/containerd/snapshots" "github.com/containerd/typeurl" "github.com/davecgh/go-spew/spew" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -35,11 +30,17 @@ import ( selinux "github.com/opencontainers/selinux/go-selinux" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd" + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/oci" + criconfig "github.com/containerd/containerd/pkg/cri/config" cio "github.com/containerd/containerd/pkg/cri/io" customopts "github.com/containerd/containerd/pkg/cri/opts" containerstore "github.com/containerd/containerd/pkg/cri/store/container" "github.com/containerd/containerd/pkg/cri/util" ctrdutil "github.com/containerd/containerd/pkg/cri/util" + "github.com/containerd/containerd/snapshots" ) func init() { @@ -186,7 +187,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations)) // Set snapshotter before any other options. opts := []containerd.NewContainerOpts{ - containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter), + containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)), // Prepare container rootfs. This is always writeable even if // the container wants a readonly rootfs since we want to give // the runtime (runc) a chance to modify (e.g. to create mount @@ -348,3 +349,14 @@ func (c *criService) runtimeSpec(id string, baseSpecFile string, opts ...oci.Spe return spec, nil } + +// Overrides the default snapshotter if Snapshotter is set for this runtime. +// See See https://github.com/containerd/containerd/issues/6657 +func (c *criService) runtimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string { + if ociRuntime.Snapshotter == "" { + return c.config.ContainerdConfig.Snapshotter + } + + log.G(ctx).Debugf("Set snapshotter for runtime %s to %s", ociRuntime.Type, ociRuntime.Snapshotter) + return ociRuntime.Snapshotter +} diff --git a/pkg/cri/server/container_create_test.go b/pkg/cri/server/container_create_test.go index 3e109360f..efe058a1b 100644 --- a/pkg/cri/server/container_create_test.go +++ b/pkg/cri/server/container_create_test.go @@ -22,13 +22,13 @@ import ( goruntime "runtime" "testing" - "github.com/containerd/containerd/oci" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/oci" "github.com/containerd/containerd/pkg/cri/config" "github.com/containerd/containerd/pkg/cri/constants" "github.com/containerd/containerd/pkg/cri/opts" @@ -416,3 +416,35 @@ func TestBaseRuntimeSpec(t *testing.T) { assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath) } + +func TestRuntimeSnapshotter(t *testing.T) { + defaultRuntime := config.Runtime{ + Snapshotter: "", + } + + fooRuntime := config.Runtime{ + Snapshotter: "devmapper", + } + + for desc, test := range map[string]struct { + runtime config.Runtime + expectSnapshotter string + }{ + "should return default snapshotter when runtime.Snapshotter is not set": { + runtime: defaultRuntime, + expectSnapshotter: config.DefaultConfig().Snapshotter, + }, + "should return overridden snapshotter when runtime.Snapshotter is set": { + runtime: fooRuntime, + expectSnapshotter: "devmapper", + }, + } { + t.Run(desc, func(t *testing.T) { + cri := newTestCRIService() + cri.config = config.Config{ + PluginConfig: config.DefaultConfig(), + } + assert.Equal(t, test.expectSnapshotter, cri.runtimeSnapshotter(context.Background(), test.runtime)) + }) + } +} diff --git a/pkg/cri/server/image_pull.go b/pkg/cri/server/image_pull.go index 71dc4b628..678d2b737 100644 --- a/pkg/cri/server/image_pull.go +++ b/pkg/cri/server/image_pull.go @@ -33,20 +33,21 @@ import ( "sync/atomic" "time" - "github.com/containerd/containerd" - "github.com/containerd/containerd/errdefs" - containerdimages "github.com/containerd/containerd/images" - "github.com/containerd/containerd/labels" - "github.com/containerd/containerd/log" - distribution "github.com/containerd/containerd/reference/docker" - "github.com/containerd/containerd/remotes/docker" - "github.com/containerd/containerd/remotes/docker/config" "github.com/containerd/imgcrypt" "github.com/containerd/imgcrypt/images/encryption" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd" + "github.com/containerd/containerd/errdefs" + containerdimages "github.com/containerd/containerd/images" + "github.com/containerd/containerd/labels" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/pkg/cri/annotations" criconfig "github.com/containerd/containerd/pkg/cri/config" + distribution "github.com/containerd/containerd/reference/docker" + "github.com/containerd/containerd/remotes/docker" + "github.com/containerd/containerd/remotes/docker/config" ) // For image management: @@ -126,10 +127,17 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) } ) + defer pcancel() + snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, ref, r.SandboxConfig) + if err != nil { + return nil, err + } + log.G(ctx).Debugf("PullImage %q with snapshotter %s", ref, snapshotter) + pullOpts := []containerd.RemoteOpt{ containerd.WithSchema1Conversion, //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. containerd.WithResolver(resolver), - containerd.WithPullSnapshotter(c.config.ContainerdConfig.Snapshotter), + containerd.WithPullSnapshotter(snapshotter), containerd.WithPullUnpack, containerd.WithPullLabel(imageLabelKey, imageLabelValue), containerd.WithMaxConcurrentDownloads(c.config.MaxConcurrentDownloads), @@ -786,3 +794,29 @@ func (rt *pullRequestReporterRoundTripper) RoundTrip(req *http.Request) (*http.R } return resp, err } + +// Given that runtime information is not passed from PullImageRequest, we depend on an experimental annotation +// passed from pod sandbox config to get the runtimeHandler. The annotation key is specified in configuration. +// Once we know the runtime, try to override default snapshotter if it is set for this runtime. +// See https://github.com/containerd/containerd/issues/6657 +func (c *criService) snapshotterFromPodSandboxConfig(ctx context.Context, imageRef string, + s *runtime.PodSandboxConfig) (string, error) { + snapshotter := c.config.ContainerdConfig.Snapshotter + if s == nil || s.Annotations == nil { + return snapshotter, nil + } + + runtimeHandler, ok := s.Annotations[annotations.RuntimeHandler] + if !ok { + return snapshotter, nil + } + + ociRuntime, err := c.getSandboxRuntime(s, runtimeHandler) + if err != nil { + return "", fmt.Errorf("experimental: failed to get sandbox runtime for %s, err: %+v", runtimeHandler, err) + } + + snapshotter = c.runtimeSnapshotter(context.Background(), ociRuntime) + log.G(ctx).Infof("experimental: PullImage %q for runtime %s, using snapshotter %s", imageRef, runtimeHandler, snapshotter) + return snapshotter, nil +} diff --git a/pkg/cri/server/image_pull_test.go b/pkg/cri/server/image_pull_test.go index 50209f37f..92ee47ead 100644 --- a/pkg/cri/server/image_pull_test.go +++ b/pkg/cri/server/image_pull_test.go @@ -28,6 +28,7 @@ import ( "github.com/stretchr/testify/assert" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/pkg/cri/annotations" criconfig "github.com/containerd/containerd/pkg/cri/config" ) @@ -377,3 +378,64 @@ func TestImageLayersLabel(t *testing.T) { }) } } + +func TestSnapshotterFromPodSandboxConfig(t *testing.T) { + defaultSnashotter := "native" + runtimeSnapshotter := "devmapper" + tests := []struct { + desc string + podSandboxConfig *runtime.PodSandboxConfig + expectSnapshotter string + expectErr error + }{ + { + desc: "should return default snapshotter for nil podSandboxConfig", + expectSnapshotter: defaultSnashotter, + }, + { + desc: "should return default snapshotter for nil podSandboxConfig.Annotations", + podSandboxConfig: &runtime.PodSandboxConfig{}, + expectSnapshotter: defaultSnashotter, + }, + { + desc: "should return default snapshotter for empty podSandboxConfig.Annotations", + podSandboxConfig: &runtime.PodSandboxConfig{ + Annotations: make(map[string]string), + }, + expectSnapshotter: defaultSnashotter, + }, + { + desc: "should return error for runtime not found", + podSandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + annotations.RuntimeHandler: "runtime-not-exists", + }, + }, + expectErr: fmt.Errorf(`experimental: failed to get sandbox runtime for runtime-not-exists, err: no runtime for "runtime-not-exists" is configured`), + expectSnapshotter: "", + }, + { + desc: "should return snapshotter provided in podSandboxConfig.Annotations", + podSandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + annotations.RuntimeHandler: "exiting-runtime", + }, + }, + expectSnapshotter: runtimeSnapshotter, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + cri := newTestCRIService() + cri.config.ContainerdConfig.Snapshotter = defaultSnashotter + cri.config.ContainerdConfig.Runtimes = make(map[string]criconfig.Runtime) + cri.config.ContainerdConfig.Runtimes["exiting-runtime"] = criconfig.Runtime{ + Snapshotter: runtimeSnapshotter, + } + snapshotter, err := cri.snapshotterFromPodSandboxConfig(context.Background(), "test-image", tt.podSandboxConfig) + assert.Equal(t, tt.expectSnapshotter, snapshotter) + assert.Equal(t, tt.expectErr, err) + }) + } +} diff --git a/pkg/cri/server/sandbox_run.go b/pkg/cri/server/sandbox_run.go index f9aefaacf..5cbe2d843 100644 --- a/pkg/cri/server/sandbox_run.go +++ b/pkg/cri/server/sandbox_run.go @@ -27,19 +27,19 @@ import ( "strings" "time" - "github.com/containerd/containerd" - containerdio "github.com/containerd/containerd/cio" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/snapshots" cni "github.com/containerd/go-cni" "github.com/containerd/nri" v1 "github.com/containerd/nri/types/v1" "github.com/containerd/typeurl" "github.com/davecgh/go-spew/spew" + selinux "github.com/opencontainers/selinux/go-selinux" "github.com/sirupsen/logrus" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd" + containerdio "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/log" "github.com/containerd/containerd/pkg/cri/annotations" criconfig "github.com/containerd/containerd/pkg/cri/config" customopts "github.com/containerd/containerd/pkg/cri/opts" @@ -48,7 +48,7 @@ import ( "github.com/containerd/containerd/pkg/cri/util" ctrdutil "github.com/containerd/containerd/pkg/cri/util" "github.com/containerd/containerd/pkg/netns" - selinux "github.com/opencontainers/selinux/go-selinux" + "github.com/containerd/containerd/snapshots" ) func init() { @@ -211,7 +211,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox } snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations)) opts := []containerd.NewContainerOpts{ - containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter), + containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)), customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt), containerd.WithSpec(spec, specOpts...), containerd.WithContainerLabels(sandboxLabels),