github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Support runtime specific configurations.

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu 2018-10-08 13:39:05 -07:00
parent e4f33828c3
commit 1442425f92
9 changed files with 257 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
docs/config.md
View File

@ -21,7 +21,10 @@ The explanation and default value of each configuration item are as follows:
# stats_collect_period is the period (in seconds) of snapshots stats collection.
stats_collect_period = 10
# systemd_cgroup enables systemd cgroup support.
# systemd_cgroup enables systemd cgroup support. This only works for runtime
# type "io.containerd.runtime.v1.linux".
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
# For runtime "io.containerd.runc.v1", use the option `SystemdCgroup`.
systemd_cgroup = false
# enable_tls_streaming enables the TLS streaming support.
@ -32,6 +35,7 @@ The explanation and default value of each configuration item are as follows:
[plugins.cri.x509_key_pair_streaming]
# tls_cert_file is the filepath to the certificate paired with the "tls_key_file"
tls_cert_file = ""
# tls_key_file is the filepath to the private key paired with the "tls_cert_file"
tls_key_file = ""
@ -46,7 +50,10 @@ The explanation and default value of each configuration item are as follows:
# snapshotter is the snapshotter used by containerd.
snapshotter = "overlayfs"
# no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
# no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc.
# This only works for runtime type "io.containerd.runtime.v1.linux".
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
# For runtime "io.containerd.runc.v1", use the option `NoPivotRoot`.
no_pivot = false
# "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
@ -55,13 +62,37 @@ The explanation and default value of each configuration item are as follows:
runtime_type = "io.containerd.runtime.v1.linux"
# runtime_engine is the name of the runtime engine used by containerd.
# This only works for runtime type "io.containerd.runtime.v1.linux".
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
# For runtime "io.containerd.runc.v1", use the option `BinaryName`.
runtime_engine = ""
# runtime_root is the directory used by containerd for runtime state.
# This only works for runtime type "io.containerd.runtime.v1.linux".
# DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
# For runtime "io.containerd.runc.v1", use the option `Root`.
runtime_root = ""
# "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
# "plugins.cri.containerd.default_runtime.options" is options specific to
# the default runtime. The options type for "io.containerd.runtime.v1.linux" is:
# https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/linux/runctypes/runc.pb.go#L40
# NOTE: when `options` is specified, all related deprecated options will
# be ignored, including `systemd_cgroup`, `no_pivot`, `runtime_engine`
# and `runtime_root`.
[plugins.cri.containerd.default_runtime.options]
# Runtime is the binary name of the runtime.
Runtime = ""
# RuntimeRoot is the root directory of the runtime.
RuntimeRoot = ""
# CriuPath is the criu binary path.
CriuPath = ""
# SystemdCgroup enables systemd cgroups.
SystemdCgroup = false
# "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
# DEPRECATED: use plugins.cri.runtimes instead. If provided, this runtime is mapped to the
# runtime handler named 'untrusted'. It is a configuration error to provide both the (now
# deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
@ -78,16 +109,41 @@ The explanation and default value of each configuration item are as follows:
# plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types
# of runtime configurations, to the matching configurations. In this example,
# 'runtime_handler_name' is the RuntimeHandler string to match.
[plugins.cri.containerd.runtimes.runtime_handler_name]
# 'runc' is the RuntimeHandler string to match.
[plugins.cri.containerd.runtimes.runc]
# runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
runtime_type = ""
runtime_type = "io.containerd.runc.v1"
# runtime_engine is the name of the runtime engine used by containerd.
runtime_engine = ""
# "plugins.cri.containerd.runtimes.runc.options" is options specific to
# "io.containerd.runc.v1". Its corresponding options type is:
# https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/v2/runc/options/oci.pb.go#L39.
[plugins.cri.containerd.runtimes.runc.options]
# NoPivotRoot disables pivot root when creating a container.
NoPivotRoot = false
# runtime_root is the directory used by containerd for runtime state.
runtime_root = ""
# NoNewKeyring disables new keyring for the container.
NoNewKeyring = false
# ShimCgroup places the shim in a cgroup.
ShimCgroup = ""
# IoUid sets the I/O's pipes uid.
IoUid = 0
# IoGid sets the I/O's pipes gid.
IoGid = 0
# BinaryName is the binary name of the runc binary.
BinaryName = ""
# Root is the runc root directory.
Root = ""
# CriuPath is the criu binary path.
CriuPath = ""
# SystemdCgroup enables systemd cgroups.
SystemdCgroup = false
# "plugins.cri.cni" contains config related to cni
[plugins.cri.cni]

16
pkg/config/config.go
View File

@ -16,7 +16,10 @@ limitations under the License.
package config
import "github.com/containerd/containerd"
import (
"github.com/BurntSushi/toml"
"github.com/containerd/containerd"
)
// Runtime struct to contain the type(ID), engine, and root variables for a default runtime
// and a runtime for untrusted worload.
@ -24,9 +27,16 @@ type Runtime struct {
// Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
Type string `toml:"runtime_type" json:"runtimeType"`
// Engine is the name of the runtime engine used by containerd.
// This only works for runtime type "io.containerd.runtime.v1.linux".
// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
Engine string `toml:"runtime_engine" json:"runtimeEngine"`
// Root is the directory used by containerd for runtime state.
// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
// This only works for runtime type "io.containerd.runtime.v1.linux".
Root string `toml:"runtime_root" json:"runtimeRoot"`
// Options are config options for the runtime. If options is loaded
// from toml config, it will be toml.Primitive.
Options *toml.Primitive `toml:"options" json:"options"`
}
// ContainerdConfig contains toml config related to containerd
@ -46,6 +56,8 @@ type ContainerdConfig struct {
// configurations, to the matching configurations.
Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"`
// NoPivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
// This only works for runtime type "io.containerd.runtime.v1.linux".
// DEPRECATED: use Runtime.Options instead. Remove when shim v1 is deprecated.
NoPivot bool `toml:"no_pivot" json:"noPivot"`
}
@ -119,6 +131,8 @@ type PluginConfig struct {
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"`
// SystemdCgroup enables systemd cgroup support.
// This only works for runtime type "io.containerd.runtime.v1.linux".
// DEPRECATED: config runc runtime handler instead. Remove when shim v1 is deprecated.
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup"`
// EnableTLSStreaming indicates to enable the TLS streaming support.
EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`

17
pkg/server/container_create.go
View File

@ -30,7 +30,6 @@ import (
"github.com/containerd/containerd/contrib/seccomp"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/runtime/linux/runctypes"
"github.com/containerd/typeurl"
"github.com/davecgh/go-spew/spew"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@ -125,11 +124,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
if err != nil {
return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID)
}
ociRuntime, err := getRuntimeConfigFromContainerInfo(sandboxInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get OCI runtime")
}
logrus.Debugf("Use OCI %+v for container %q", ociRuntime, id)
// Create container root directory.
containerRootDir := c.getContainerRootDir(id)
@ -261,14 +255,13 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
}
containerLabels := buildLabels(config.Labels, containerKindContainer)
runtimeOptions, err := getRuntimeOptions(sandboxInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get runtime options")
}
opts = append(opts,
containerd.WithSpec(spec, specOpts...),
containerd.WithRuntime(
ociRuntime.Type,
&runctypes.RuncOptions{
Runtime: ociRuntime.Engine,
RuntimeRoot: ociRuntime.Root,
SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause
containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions),
containerd.WithContainerLabels(containerLabels),
containerd.WithContainerExtension(containerMetadataExtension, &meta))
var cntr containerd.Container

8
pkg/server/container_start.go
View File

@ -108,8 +108,14 @@ func (c *criService) startContainer(ctx context.Context,
return cntr.IO, nil
}
ctrInfo, err := container.Info(ctx)
if err != nil {
return errors.Wrap(err, "failed to get container info")
}
var taskOpts []containerd.NewTaskOpts
if c.config.NoPivot {
// TODO(random-liu): Remove this after shim v1 is deprecated.
if c.config.NoPivot && ctrInfo.Runtime.Name == linuxRuntime {
taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
}
task, err := container.NewTask(ctx, ioCreation, taskOpts...)

11
pkg/server/container_status.go
View File

@ -24,7 +24,6 @@ import (
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
containerstore "github.com/containerd/cri/pkg/store/container"
)
@ -107,7 +106,8 @@ type containerInfo struct {
Removing bool `json:"removing"`
SnapshotKey string `json:"snapshotKey"`
Snapshotter string `json:"snapshotter"`
Runtime *criconfig.Runtime `json:"runtime"`
RuntimeType string `json:"runtimeType"`
RuntimeOptions interface{} `json:"runtimeOptions"`
Config *runtime.ContainerConfig `json:"config"`
RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"`
}
@ -142,11 +142,12 @@ func toCRIContainerInfo(ctx context.Context, container containerstore.Container,
ci.SnapshotKey = ctrInfo.SnapshotKey
ci.Snapshotter = ctrInfo.Snapshotter
ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo)
runtimeOptions, err := getRuntimeOptions(ctrInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get container runtime config")
return nil, errors.Wrap(err, "failed to get runtime options")
}
ci.Runtime = &ociRuntime
ci.RuntimeType = ctrInfo.Runtime.Name
ci.RuntimeOptions = runtimeOptions
infoBytes, err := json.Marshal(ci)
if err != nil {

72
pkg/server/helpers.go
View File

@ -25,8 +25,10 @@ import (
"strconv"
"strings"
"github.com/BurntSushi/toml"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/runtime/linux/runctypes"
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
"github.com/containerd/typeurl"
"github.com/docker/distribution/reference"
imagedigest "github.com/opencontainers/go-digest"
@ -123,6 +125,14 @@ const (
networkAttachCount = 2
)
// Runtime type strings for various runtimes.
const (
// linuxRuntime is the legacy linux runtime for shim v1.
linuxRuntime = "io.containerd.runtime.v1.linux"
// runcRuntime is the runc runtime for shim v2.
runcRuntime = "io.containerd.runc.v1"
)
// makeSandboxName generates sandbox name from sandbox metadata. The name
// generated is unique as long as sandbox metadata is unique.
func makeSandboxName(s *runtime.PodSandboxMetadata) string {
@ -390,26 +400,6 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str
}
}
// getRuntimeConfigFromContainerInfo gets runtime configuration from containerd
// container info.
func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtime, error) {
r := criconfig.Runtime{
Type: c.Runtime.Name,
}
if c.Runtime.Options == nil {
// CRI plugin makes sure that runtime option is always set.
return criconfig.Runtime{}, errors.New("runtime options is nil")
}
data, err := typeurl.UnmarshalAny(c.Runtime.Options)
if err != nil {
return criconfig.Runtime{}, errors.Wrap(err, "failed to unmarshal runtime options")
}
runtimeOpts := data.(*runctypes.RuncOptions)
r.Engine = runtimeOpts.Runtime
r.Root = runtimeOpts.RuntimeRoot
return r, nil
}
// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
return &runtime.AuthConfig{
@ -464,3 +454,45 @@ func parseImageReferences(refs []string) ([]string, []string) {
}
return tags, digests
}
// generateRuntimeOptions generates runtime options from cri plugin config.
func generateRuntimeOptions(r criconfig.Runtime, c criconfig.Config) (interface{}, error) {
if r.Options == nil {
if r.Type != linuxRuntime {
return nil, nil
}
// This is a legacy config, generate runctypes.RuncOptions.
return &runctypes.RuncOptions{
Runtime: r.Engine,
RuntimeRoot: r.Root,
SystemdCgroup: c.SystemdCgroup,
}, nil
}
options := getRuntimeOptionsType(r.Type)
if err := toml.PrimitiveDecode(*r.Options, options); err != nil {
return nil, err
}
return options, nil
}
// getRuntimeOptionsType gets empty runtime options by the runtime type name.
func getRuntimeOptionsType(t string) interface{} {
switch t {
case runcRuntime:
return &runcoptions.Options{}
default:
return &runctypes.RuncOptions{}
}
}
// getRuntimeOptions get runtime options from container metadata.
func getRuntimeOptions(c containers.Container) (interface{}, error) {
if c.Runtime.Options == nil {
return nil, nil
}
opts, err := typeurl.UnmarshalAny(c.Runtime.Options)
if err != nil {
return nil, err
}
return opts, nil
}

126
pkg/server/helpers_test.go
View File

@ -20,12 +20,12 @@ import (
"sort"
"testing"
"github.com/containerd/containerd"
"github.com/containerd/containerd/containers"
"github.com/BurntSushi/toml"
"github.com/containerd/containerd/runtime/linux/runctypes"
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
imagedigest "github.com/opencontainers/go-digest"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
"github.com/stretchr/testify/require"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
@ -152,49 +152,6 @@ func TestBuildLabels(t *testing.T) {
assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label")
}
func TestGetRuntimeConfigFromContainerInfo(t *testing.T) {
for desc, test := range map[string]struct {
typ string
engine string
root string
expectErr bool
expectedRuntime criconfig.Runtime
}{
"should return error if there is no runc options": {
typ: "test.type",
expectErr: true,
},
"should retrieve runtime from container info": {
typ: "test.type",
engine: "test-engine",
root: "/test/root",
expectedRuntime: criconfig.Runtime{
Type: "test.type",
Engine: "test-engine",
Root: "/test/root",
},
},
} {
t.Run(desc, func(t *testing.T) {
var opts interface{}
if test.engine != "" || test.root != "" {
opts = &runctypes.RuncOptions{
Runtime: test.engine,
RuntimeRoot: test.root,
}
}
c := containers.Container{}
assert.NoError(t, containerd.WithRuntime(
test.typ,
opts,
)(context.Background(), nil, &c))
r, err := getRuntimeConfigFromContainerInfo(c)
assert.Equal(t, test.expectErr, err != nil)
assert.Equal(t, test.expectedRuntime, r)
})
}
}
func TestOrderedMounts(t *testing.T) {
mounts := []*runtime.Mount{
{ContainerPath: "/a/b/c"},
@ -270,3 +227,80 @@ func TestLocalResolve(t *testing.T) {
assert.Equal(t, store.ErrNotExist, err)
assert.Equal(t, imagestore.Image{}, img)
}
func TestGenerateRuntimeOptions(t *testing.T) {
nilOpts := `
systemd_cgroup = true
[containerd]
no_pivot = true
[containerd.default_runtime]
runtime_type = "` + linuxRuntime + `"
[containerd.runtimes.runc]
runtime_type = "` + runcRuntime + `"
`
nonNilOpts := `
systemd_cgroup = true
[containerd]
no_pivot = true
[containerd.default_runtime]
runtime_type = "` + linuxRuntime + `"
[containerd.default_runtime.options]
Runtime = "default"
RuntimeRoot = "/default"
[containerd.runtimes.runc]
runtime_type = "` + runcRuntime + `"
[containerd.runtimes.runc.options]
BinaryName = "runc"
Root = "/runc"
NoNewKeyring = true
`
var nilOptsConfig, nonNilOptsConfig criconfig.Config
_, err := toml.Decode(nilOpts, &nilOptsConfig)
require.NoError(t, err)
_, err = toml.Decode(nonNilOpts, &nonNilOptsConfig)
require.NoError(t, err)
require.Len(t, nilOptsConfig.Runtimes, 1)
require.Len(t, nonNilOptsConfig.Runtimes, 1)
for desc, test := range map[string]struct {
r criconfig.Runtime
c criconfig.Config
expectedOptions interface{}
}{
"when options is nil, should return nil option for non legacy runtime": {
r: nilOptsConfig.Runtimes["runc"],
c: nilOptsConfig,
expectedOptions: nil,
},
"when options is nil, should use legacy fields for legacy runtime": {
r: nilOptsConfig.DefaultRuntime,
c: nilOptsConfig,
expectedOptions: &runctypes.RuncOptions{
SystemdCgroup: true,
},
},
"when options is not nil, should be able to decode for io.containerd.runc.v1": {
r: nonNilOptsConfig.Runtimes["runc"],
c: nonNilOptsConfig,
expectedOptions: &runcoptions.Options{
BinaryName: "runc",
Root: "/runc",
NoNewKeyring: true,
},
},
"when options is not nil, should be able to decode for legacy runtime": {
r: nonNilOptsConfig.DefaultRuntime,
c: nonNilOptsConfig,
expectedOptions: &runctypes.RuncOptions{
Runtime: "default",
RuntimeRoot: "/default",
},
},
} {
t.Run(desc, func(t *testing.T) {
opts, err := generateRuntimeOptions(test.r, test.c)
assert.NoError(t, err)
assert.Equal(t, test.expectedOptions, opts)
})
}
}

17
pkg/server/sandbox_run.go
View File

@ -26,8 +26,7 @@ import (
containerdio "github.com/containerd/containerd/cio"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/runtime/linux/runctypes"
"github.com/containerd/go-cni"
cni "github.com/containerd/go-cni"
"github.com/containerd/typeurl"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
@ -172,18 +171,17 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
sandboxLabels := buildLabels(config.Labels, containerKindSandbox)
runtimeOpts, err := generateRuntimeOptions(ociRuntime, c.config)
if err != nil {
return nil, errors.Wrap(err, "failed to generate runtime options")
}
opts := []containerd.NewContainerOpts{
containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
customopts.WithNewSnapshot(id, image.Image),
containerd.WithSpec(spec, specOpts...),
containerd.WithContainerLabels(sandboxLabels),
containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata),
containerd.WithRuntime(
ociRuntime.Type,
&runctypes.RuncOptions{
Runtime: ociRuntime.Engine,
RuntimeRoot: ociRuntime.Root,
SystemdCgroup: c.config.SystemdCgroup})} // TODO (mikebrow): add CriuPath when we add support for pause
containerd.WithRuntime(ociRuntime.Type, runtimeOpts)}
container, err := c.client.NewContainer(ctx, id, opts...)
if err != nil {
@ -297,7 +295,8 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
id, name)
var taskOpts []containerd.NewTaskOpts
if c.config.NoPivot {
// TODO(random-liu): Remove this after shim v1 is deprecated.
if c.config.NoPivot && ociRuntime.Type == linuxRuntime {
taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
}
// We don't need stdio for sandbox container.

11
pkg/server/sandbox_status.go
View File

@ -26,7 +26,6 @@ import (
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
)
@ -109,7 +108,8 @@ type sandboxInfo struct {
SnapshotKey string `json:"snapshotKey"`
Snapshotter string `json:"snapshotter"`
RuntimeHandler string `json:"runtimeHandler"`
Runtime *criconfig.Runtime `json:"runtime"`
RuntimeType string `json:"runtimeType"`
RuntimeOptions interface{} `json:"runtimeOptions"`
Config *runtime.PodSandboxConfig `json:"config"`
RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"`
}
@ -167,11 +167,12 @@ func toCRISandboxInfo(ctx context.Context, sandbox sandboxstore.Sandbox) (map[st
si.SnapshotKey = ctrInfo.SnapshotKey
si.Snapshotter = ctrInfo.Snapshotter
ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo)
runtimeOptions, err := getRuntimeOptions(ctrInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get sandbox container runtime config")
return nil, errors.Wrap(err, "failed to get runtime options")
}
si.Runtime = &ociRuntime
si.RuntimeType = ctrInfo.Runtime.Name
si.RuntimeOptions = runtimeOptions
infoBytes, err := json.Marshal(si)
if err != nil {