github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #6899 from shuaichang/ISSUE6657-support-runtime-snapshotter

Browse Source 
Support runtime level snapshotter for issue 6657
This commit is contained in:
Phil Estes 2022-06-03 10:04:53 +02:00 committed by GitHub
commit 2b661b890f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 182 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/cri/config.md
View File

@ -228,7 +228,8 @@ version = 2
# 'plugins."io.containerd.grpc.v1.cri".containerd' contains config related to containerd
[plugins."io.containerd.grpc.v1.cri".containerd]
# snapshotter is the snapshotter used by containerd.
# snapshotter is the default snapshotter used by containerd
# for all runtimes, if not overridden by an experimental runtime's snapshotter config.
snapshotter = "overlayfs"
# no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc.
@ -329,6 +330,11 @@ version = 2
# config files being loaded from the CNI config directory.
cni_max_conf_num = 1
# snapshotter overrides the global default snapshotter to a runtime specific value.
# Please be aware that overriding the default snapshotter on a runtime basis is currently an experimental feature.
# See https://github.com/containerd/containerd/issues/6657 for context.
snapshotter = ""
# 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to
# "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is:
# https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 .

7
pkg/cri/annotations/annotations.go
View File

@ -70,6 +70,13 @@ const (
// PodAnnotations are the annotations of the pod
PodAnnotations = "io.kubernetes.cri.pod-annotations"
// RuntimeHandler an experimental annotation key for getting runtime handler from pod annotations.
// See https://github.com/containerd/containerd/issues/6657 and https://github.com/containerd/containerd/pull/6899 for details.
// The value of this annotation should be the runtime for sandboxes.
// e.g. for [plugins.cri.containerd.runtimes.runc] runtime config, this value should be runc
// TODO: we should deprecate this annotation as soon as kubelet supports passing RuntimeHandler from PullImageRequest
RuntimeHandler = "io.containerd.cri.runtime-handler"
// WindowsHostProcess is used by hcsshim to identify windows pods that are running HostProcesses
WindowsHostProcess = "microsoft.com/hostprocess-container"
)

5
pkg/cri/config/config.go
View File

@ -71,6 +71,11 @@ type Runtime struct {
// be loaded from the cni config directory by go-cni. Set the value to 0 to
// load all config files (no arbitrary limit). The legacy default value is 1.
NetworkPluginMaxConfNum int `toml:"cni_max_conf_num" json:"cniMaxConfNum"`
// Snapshotter setting snapshotter at runtime level instead of making it as a global configuration.
// An example use case is to use devmapper or other snapshotters in Kata containers for performance and security
// while using default snapshotters for operational simplicity.
// See https://github.com/containerd/containerd/issues/6657 for details.
Snapshotter string `toml:"snapshotter" json:"snapshotter"`
}
// ContainerdConfig contains toml config related to containerd

24
pkg/cri/server/container_create.go
View File

@ -23,11 +23,6 @@ import (
"path/filepath"
"time"
"github.com/containerd/containerd"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/typeurl"
"github.com/davecgh/go-spew/spew"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@ -35,11 +30,17 @@ import (
selinux "github.com/opencontainers/selinux/go-selinux"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/oci"
criconfig "github.com/containerd/containerd/pkg/cri/config"
cio "github.com/containerd/containerd/pkg/cri/io"
customopts "github.com/containerd/containerd/pkg/cri/opts"
containerstore "github.com/containerd/containerd/pkg/cri/store/container"
"github.com/containerd/containerd/pkg/cri/util"
ctrdutil "github.com/containerd/containerd/pkg/cri/util"
"github.com/containerd/containerd/snapshots"
)
func init() {
@ -186,7 +187,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations))
// Set snapshotter before any other options.
opts := []containerd.NewContainerOpts{
containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)),
// Prepare container rootfs. This is always writeable even if
// the container wants a readonly rootfs since we want to give
// the runtime (runc) a chance to modify (e.g. to create mount
@ -348,3 +349,14 @@ func (c *criService) runtimeSpec(id string, baseSpecFile string, opts ...oci.Spe
return spec, nil
}
// Overrides the default snapshotter if Snapshotter is set for this runtime.
// See See https://github.com/containerd/containerd/issues/6657
func (c *criService) runtimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string {
if ociRuntime.Snapshotter == "" {
return c.config.ContainerdConfig.Snapshotter
}
log.G(ctx).Debugf("Set snapshotter for runtime %s to %s", ociRuntime.Type, ociRuntime.Snapshotter)
return ociRuntime.Snapshotter
}

34
pkg/cri/server/container_create_test.go
View File

@ -22,13 +22,13 @@ import (
goruntime "runtime"
"testing"
"github.com/containerd/containerd/oci"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/pkg/cri/config"
"github.com/containerd/containerd/pkg/cri/constants"
"github.com/containerd/containerd/pkg/cri/opts"
@ -418,3 +418,35 @@ func TestBaseRuntimeSpec(t *testing.T) {
assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
}
func TestRuntimeSnapshotter(t *testing.T) {
defaultRuntime := config.Runtime{
Snapshotter: "",
}
fooRuntime := config.Runtime{
Snapshotter: "devmapper",
}
for desc, test := range map[string]struct {
runtime config.Runtime
expectSnapshotter string
}{
"should return default snapshotter when runtime.Snapshotter is not set": {
runtime: defaultRuntime,
expectSnapshotter: config.DefaultConfig().Snapshotter,
},
"should return overridden snapshotter when runtime.Snapshotter is set": {
runtime: fooRuntime,
expectSnapshotter: "devmapper",
},
} {
t.Run(desc, func(t *testing.T) {
cri := newTestCRIService()
cri.config = config.Config{
PluginConfig: config.DefaultConfig(),
}
assert.Equal(t, test.expectSnapshotter, cri.runtimeSnapshotter(context.Background(), test.runtime))
})
}
}

52
pkg/cri/server/image_pull.go
View File

@ -33,20 +33,21 @@ import (
"sync/atomic"
"time"
"github.com/containerd/containerd"
"github.com/containerd/containerd/errdefs"
containerdimages "github.com/containerd/containerd/images"
"github.com/containerd/containerd/labels"
"github.com/containerd/containerd/log"
distribution "github.com/containerd/containerd/reference/docker"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/remotes/docker/config"
"github.com/containerd/imgcrypt"
"github.com/containerd/imgcrypt/images/encryption"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd"
"github.com/containerd/containerd/errdefs"
containerdimages "github.com/containerd/containerd/images"
"github.com/containerd/containerd/labels"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
distribution "github.com/containerd/containerd/reference/docker"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/remotes/docker/config"
)
// For image management:
@ -126,10 +127,17 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest)
}
)
defer pcancel()
snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, ref, r.SandboxConfig)
if err != nil {
return nil, err
}
log.G(ctx).Debugf("PullImage %q with snapshotter %s", ref, snapshotter)
pullOpts := []containerd.RemoteOpt{
containerd.WithSchema1Conversion, //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
containerd.WithResolver(resolver),
containerd.WithPullSnapshotter(c.config.ContainerdConfig.Snapshotter),
containerd.WithPullSnapshotter(snapshotter),
containerd.WithPullUnpack,
containerd.WithPullLabel(imageLabelKey, imageLabelValue),
containerd.WithMaxConcurrentDownloads(c.config.MaxConcurrentDownloads),
@ -786,3 +794,29 @@ func (rt *pullRequestReporterRoundTripper) RoundTrip(req *http.Request) (*http.R
}
return resp, err
}
// Given that runtime information is not passed from PullImageRequest, we depend on an experimental annotation
// passed from pod sandbox config to get the runtimeHandler. The annotation key is specified in configuration.
// Once we know the runtime, try to override default snapshotter if it is set for this runtime.
// See https://github.com/containerd/containerd/issues/6657
func (c *criService) snapshotterFromPodSandboxConfig(ctx context.Context, imageRef string,
s *runtime.PodSandboxConfig) (string, error) {
snapshotter := c.config.ContainerdConfig.Snapshotter
if s == nil || s.Annotations == nil {
return snapshotter, nil
}
runtimeHandler, ok := s.Annotations[annotations.RuntimeHandler]
if !ok {
return snapshotter, nil
}
ociRuntime, err := c.getSandboxRuntime(s, runtimeHandler)
if err != nil {
return "", fmt.Errorf("experimental: failed to get sandbox runtime for %s, err: %+v", runtimeHandler, err)
}
snapshotter = c.runtimeSnapshotter(context.Background(), ociRuntime)
log.G(ctx).Infof("experimental: PullImage %q for runtime %s, using snapshotter %s", imageRef, runtimeHandler, snapshotter)
return snapshotter, nil
}

62
pkg/cri/server/image_pull_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
)
@ -381,3 +382,64 @@ func TestImageLayersLabel(t *testing.T) {
})
}
}
func TestSnapshotterFromPodSandboxConfig(t *testing.T) {
defaultSnashotter := "native"
runtimeSnapshotter := "devmapper"
tests := []struct {
desc string
podSandboxConfig *runtime.PodSandboxConfig
expectSnapshotter string
expectErr error
}{
{
desc: "should return default snapshotter for nil podSandboxConfig",
expectSnapshotter: defaultSnashotter,
},
{
desc: "should return default snapshotter for nil podSandboxConfig.Annotations",
podSandboxConfig: &runtime.PodSandboxConfig{},
expectSnapshotter: defaultSnashotter,
},
{
desc: "should return default snapshotter for empty podSandboxConfig.Annotations",
podSandboxConfig: &runtime.PodSandboxConfig{
Annotations: make(map[string]string),
},
expectSnapshotter: defaultSnashotter,
},
{
desc: "should return error for runtime not found",
podSandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.RuntimeHandler: "runtime-not-exists",
},
},
expectErr: fmt.Errorf(`experimental: failed to get sandbox runtime for runtime-not-exists, err: no runtime for "runtime-not-exists" is configured`),
expectSnapshotter: "",
},
{
desc: "should return snapshotter provided in podSandboxConfig.Annotations",
podSandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.RuntimeHandler: "exiting-runtime",
},
},
expectSnapshotter: runtimeSnapshotter,
},
}
for _, tt := range tests {
t.Run(tt.desc, func(t *testing.T) {
cri := newTestCRIService()
cri.config.ContainerdConfig.Snapshotter = defaultSnashotter
cri.config.ContainerdConfig.Runtimes = make(map[string]criconfig.Runtime)
cri.config.ContainerdConfig.Runtimes["exiting-runtime"] = criconfig.Runtime{
Snapshotter: runtimeSnapshotter,
}
snapshotter, err := cri.snapshotterFromPodSandboxConfig(context.Background(), "test-image", tt.podSandboxConfig)
assert.Equal(t, tt.expectSnapshotter, snapshotter)
assert.Equal(t, tt.expectErr, err)
})
}
}

14
pkg/cri/server/sandbox_run.go
View File

@ -27,19 +27,19 @@ import (
"strings"
"time"
"github.com/containerd/containerd"
containerdio "github.com/containerd/containerd/cio"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/snapshots"
cni "github.com/containerd/go-cni"
"github.com/containerd/nri"
v1 "github.com/containerd/nri/types/v1"
"github.com/containerd/typeurl"
"github.com/davecgh/go-spew/spew"
selinux "github.com/opencontainers/selinux/go-selinux"
"github.com/sirupsen/logrus"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd"
containerdio "github.com/containerd/containerd/cio"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
customopts "github.com/containerd/containerd/pkg/cri/opts"
@ -48,7 +48,7 @@ import (
"github.com/containerd/containerd/pkg/cri/util"
ctrdutil "github.com/containerd/containerd/pkg/cri/util"
"github.com/containerd/containerd/pkg/netns"
selinux "github.com/opencontainers/selinux/go-selinux"
"github.com/containerd/containerd/snapshots"
)
func init() {
@ -211,7 +211,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
}
snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations))
opts := []containerd.NewContainerOpts{
containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)),
customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt),
containerd.WithSpec(spec, specOpts...),
containerd.WithContainerLabels(sandboxLabels),