github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #943 from Random-Liu/support-per-runtime-config

Browse Source 
Support runtime specific configurations.
This commit is contained in:
Lantao Liu
2018-10-09 08:09:12 -07:00
committed by GitHub
parent 8b0d53c09c 1442425f92
commit 70da14e4b3
9 changed files with 257 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
pkg/config/config.go
View File

@@ -16,7 +16,10 @@ limitations under the License.
package config
import "github.com/containerd/containerd"
import (
"github.com/BurntSushi/toml"
"github.com/containerd/containerd"
)
// Runtime struct to contain the type(ID), engine, and root variables for a default runtime
// and a runtime for untrusted worload.
@@ -24,9 +27,16 @@ type Runtime struct {
// Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
Type string `toml:"runtime_type" json:"runtimeType"`
// Engine is the name of the runtime engine used by containerd.
// This only works for runtime type "io.containerd.runtime.v1.linux".
// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
Engine string `toml:"runtime_engine" json:"runtimeEngine"`
// Root is the directory used by containerd for runtime state.
// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
// This only works for runtime type "io.containerd.runtime.v1.linux".
Root string `toml:"runtime_root" json:"runtimeRoot"`
// Options are config options for the runtime. If options is loaded
// from toml config, it will be toml.Primitive.
Options *toml.Primitive `toml:"options" json:"options"`
}
// ContainerdConfig contains toml config related to containerd
@@ -46,6 +56,8 @@ type ContainerdConfig struct {
// configurations, to the matching configurations.
Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"`
// NoPivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
// This only works for runtime type "io.containerd.runtime.v1.linux".
// DEPRECATED: use Runtime.Options instead. Remove when shim v1 is deprecated.
NoPivot bool `toml:"no_pivot" json:"noPivot"`
}
@@ -119,6 +131,8 @@ type PluginConfig struct {
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"`
// SystemdCgroup enables systemd cgroup support.
// This only works for runtime type "io.containerd.runtime.v1.linux".
// DEPRECATED: config runc runtime handler instead. Remove when shim v1 is deprecated.
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup"`
// EnableTLSStreaming indicates to enable the TLS streaming support.
EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`

17
pkg/server/container_create.go
View File

@@ -30,7 +30,6 @@ import (
"github.com/containerd/containerd/contrib/seccomp"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/runtime/linux/runctypes"
"github.com/containerd/typeurl"
"github.com/davecgh/go-spew/spew"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -125,11 +124,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
if err != nil {
return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID)
}
ociRuntime, err := getRuntimeConfigFromContainerInfo(sandboxInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get OCI runtime")
}
logrus.Debugf("Use OCI %+v for container %q", ociRuntime, id)
// Create container root directory.
containerRootDir := c.getContainerRootDir(id)
@@ -261,14 +255,13 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
}
containerLabels := buildLabels(config.Labels, containerKindContainer)
runtimeOptions, err := getRuntimeOptions(sandboxInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get runtime options")
}
opts = append(opts,
containerd.WithSpec(spec, specOpts...),
containerd.WithRuntime(
ociRuntime.Type,
&runctypes.RuncOptions{
Runtime: ociRuntime.Engine,
RuntimeRoot: ociRuntime.Root,
SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause
containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions),
containerd.WithContainerLabels(containerLabels),
containerd.WithContainerExtension(containerMetadataExtension, &meta))
var cntr containerd.Container

8
pkg/server/container_start.go
View File

@@ -108,8 +108,14 @@ func (c *criService) startContainer(ctx context.Context,
return cntr.IO, nil
}
ctrInfo, err := container.Info(ctx)
if err != nil {
return errors.Wrap(err, "failed to get container info")
}
var taskOpts []containerd.NewTaskOpts
if c.config.NoPivot {
// TODO(random-liu): Remove this after shim v1 is deprecated.
if c.config.NoPivot && ctrInfo.Runtime.Name == linuxRuntime {
taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
}
task, err := container.NewTask(ctx, ioCreation, taskOpts...)

25
pkg/server/container_status.go
View File

@@ -24,7 +24,6 @@ import (
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
containerstore "github.com/containerd/cri/pkg/store/container"
)
@@ -102,14 +101,15 @@ func toCRIContainerStatus(container containerstore.Container, spec *runtime.Imag
type containerInfo struct {
// TODO(random-liu): Add sandboxID in CRI container status.
SandboxID string `json:"sandboxID"`
Pid uint32 `json:"pid"`
Removing bool `json:"removing"`
SnapshotKey string `json:"snapshotKey"`
Snapshotter string `json:"snapshotter"`
Runtime *criconfig.Runtime `json:"runtime"`
Config *runtime.ContainerConfig `json:"config"`
RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"`
SandboxID string `json:"sandboxID"`
Pid uint32 `json:"pid"`
Removing bool `json:"removing"`
SnapshotKey string `json:"snapshotKey"`
Snapshotter string `json:"snapshotter"`
RuntimeType string `json:"runtimeType"`
RuntimeOptions interface{} `json:"runtimeOptions"`
Config *runtime.ContainerConfig `json:"config"`
RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"`
}
// toCRIContainerInfo converts internal container object information to CRI container status response info map.
@@ -142,11 +142,12 @@ func toCRIContainerInfo(ctx context.Context, container containerstore.Container,
ci.SnapshotKey = ctrInfo.SnapshotKey
ci.Snapshotter = ctrInfo.Snapshotter
ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo)
runtimeOptions, err := getRuntimeOptions(ctrInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get container runtime config")
return nil, errors.Wrap(err, "failed to get runtime options")
}
ci.Runtime = &ociRuntime
ci.RuntimeType = ctrInfo.Runtime.Name
ci.RuntimeOptions = runtimeOptions
infoBytes, err := json.Marshal(ci)
if err != nil {

72
pkg/server/helpers.go
View File

@@ -25,8 +25,10 @@ import (
"strconv"
"strings"
"github.com/BurntSushi/toml"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/runtime/linux/runctypes"
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
"github.com/containerd/typeurl"
"github.com/docker/distribution/reference"
imagedigest "github.com/opencontainers/go-digest"
@@ -123,6 +125,14 @@ const (
networkAttachCount = 2
)
// Runtime type strings for various runtimes.
const (
// linuxRuntime is the legacy linux runtime for shim v1.
linuxRuntime = "io.containerd.runtime.v1.linux"
// runcRuntime is the runc runtime for shim v2.
runcRuntime = "io.containerd.runc.v1"
)
// makeSandboxName generates sandbox name from sandbox metadata. The name
// generated is unique as long as sandbox metadata is unique.
func makeSandboxName(s *runtime.PodSandboxMetadata) string {
@@ -390,26 +400,6 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str
}
}
// getRuntimeConfigFromContainerInfo gets runtime configuration from containerd
// container info.
func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtime, error) {
r := criconfig.Runtime{
Type: c.Runtime.Name,
}
if c.Runtime.Options == nil {
// CRI plugin makes sure that runtime option is always set.
return criconfig.Runtime{}, errors.New("runtime options is nil")
}
data, err := typeurl.UnmarshalAny(c.Runtime.Options)
if err != nil {
return criconfig.Runtime{}, errors.Wrap(err, "failed to unmarshal runtime options")
}
runtimeOpts := data.(*runctypes.RuncOptions)
r.Engine = runtimeOpts.Runtime
r.Root = runtimeOpts.RuntimeRoot
return r, nil
}
// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
return &runtime.AuthConfig{
@@ -464,3 +454,45 @@ func parseImageReferences(refs []string) ([]string, []string) {
}
return tags, digests
}
// generateRuntimeOptions generates runtime options from cri plugin config.
func generateRuntimeOptions(r criconfig.Runtime, c criconfig.Config) (interface{}, error) {
if r.Options == nil {
if r.Type != linuxRuntime {
return nil, nil
}
// This is a legacy config, generate runctypes.RuncOptions.
return &runctypes.RuncOptions{
Runtime: r.Engine,
RuntimeRoot: r.Root,
SystemdCgroup: c.SystemdCgroup,
}, nil
}
options := getRuntimeOptionsType(r.Type)
if err := toml.PrimitiveDecode(*r.Options, options); err != nil {
return nil, err
}
return options, nil
}
// getRuntimeOptionsType gets empty runtime options by the runtime type name.
func getRuntimeOptionsType(t string) interface{} {
switch t {
case runcRuntime:
return &runcoptions.Options{}
default:
return &runctypes.RuncOptions{}
}
}
// getRuntimeOptions get runtime options from container metadata.
func getRuntimeOptions(c containers.Container) (interface{}, error) {
if c.Runtime.Options == nil {
return nil, nil
}
opts, err := typeurl.UnmarshalAny(c.Runtime.Options)
if err != nil {
return nil, err
}
return opts, nil
}

126
pkg/server/helpers_test.go
View File

@@ -20,12 +20,12 @@ import (
"sort"
"testing"
"github.com/containerd/containerd"
"github.com/containerd/containerd/containers"
"github.com/BurntSushi/toml"
"github.com/containerd/containerd/runtime/linux/runctypes"
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
imagedigest "github.com/opencontainers/go-digest"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
"github.com/stretchr/testify/require"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
@@ -152,49 +152,6 @@ func TestBuildLabels(t *testing.T) {
assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label")
}
func TestGetRuntimeConfigFromContainerInfo(t *testing.T) {
for desc, test := range map[string]struct {
typ string
engine string
root string
expectErr bool
expectedRuntime criconfig.Runtime
}{
"should return error if there is no runc options": {
typ: "test.type",
expectErr: true,
},
"should retrieve runtime from container info": {
typ: "test.type",
engine: "test-engine",
root: "/test/root",
expectedRuntime: criconfig.Runtime{
Type: "test.type",
Engine: "test-engine",
Root: "/test/root",
},
},
} {
t.Run(desc, func(t *testing.T) {
var opts interface{}
if test.engine != "" || test.root != "" {
opts = &runctypes.RuncOptions{
Runtime: test.engine,
RuntimeRoot: test.root,
}
}
c := containers.Container{}
assert.NoError(t, containerd.WithRuntime(
test.typ,
opts,
)(context.Background(), nil, &c))
r, err := getRuntimeConfigFromContainerInfo(c)
assert.Equal(t, test.expectErr, err != nil)
assert.Equal(t, test.expectedRuntime, r)
})
}
}
func TestOrderedMounts(t *testing.T) {
mounts := []*runtime.Mount{
{ContainerPath: "/a/b/c"},
@@ -270,3 +227,80 @@ func TestLocalResolve(t *testing.T) {
assert.Equal(t, store.ErrNotExist, err)
assert.Equal(t, imagestore.Image{}, img)
}
func TestGenerateRuntimeOptions(t *testing.T) {
nilOpts := `
systemd_cgroup = true
[containerd]
no_pivot = true
[containerd.default_runtime]
runtime_type = "` + linuxRuntime + `"
[containerd.runtimes.runc]
runtime_type = "` + runcRuntime + `"
`
nonNilOpts := `
systemd_cgroup = true
[containerd]
no_pivot = true
[containerd.default_runtime]
runtime_type = "` + linuxRuntime + `"
[containerd.default_runtime.options]
Runtime = "default"
RuntimeRoot = "/default"
[containerd.runtimes.runc]
runtime_type = "` + runcRuntime + `"
[containerd.runtimes.runc.options]
BinaryName = "runc"
Root = "/runc"
NoNewKeyring = true
`
var nilOptsConfig, nonNilOptsConfig criconfig.Config
_, err := toml.Decode(nilOpts, &nilOptsConfig)
require.NoError(t, err)
_, err = toml.Decode(nonNilOpts, &nonNilOptsConfig)
require.NoError(t, err)
require.Len(t, nilOptsConfig.Runtimes, 1)
require.Len(t, nonNilOptsConfig.Runtimes, 1)
for desc, test := range map[string]struct {
r criconfig.Runtime
c criconfig.Config
expectedOptions interface{}
}{
"when options is nil, should return nil option for non legacy runtime": {
r: nilOptsConfig.Runtimes["runc"],
c: nilOptsConfig,
expectedOptions: nil,
},
"when options is nil, should use legacy fields for legacy runtime": {
r: nilOptsConfig.DefaultRuntime,
c: nilOptsConfig,
expectedOptions: &runctypes.RuncOptions{
SystemdCgroup: true,
},
},
"when options is not nil, should be able to decode for io.containerd.runc.v1": {
r: nonNilOptsConfig.Runtimes["runc"],
c: nonNilOptsConfig,
expectedOptions: &runcoptions.Options{
BinaryName: "runc",
Root: "/runc",
NoNewKeyring: true,
},
},
"when options is not nil, should be able to decode for legacy runtime": {
r: nonNilOptsConfig.DefaultRuntime,
c: nonNilOptsConfig,
expectedOptions: &runctypes.RuncOptions{
Runtime: "default",
RuntimeRoot: "/default",
},
},
} {
t.Run(desc, func(t *testing.T) {
opts, err := generateRuntimeOptions(test.r, test.c)
assert.NoError(t, err)
assert.Equal(t, test.expectedOptions, opts)
})
}
}

17
pkg/server/sandbox_run.go
View File

@@ -26,8 +26,7 @@ import (
containerdio "github.com/containerd/containerd/cio"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/runtime/linux/runctypes"
"github.com/containerd/go-cni"
cni "github.com/containerd/go-cni"
"github.com/containerd/typeurl"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
@@ -172,18 +171,17 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
sandboxLabels := buildLabels(config.Labels, containerKindSandbox)
runtimeOpts, err := generateRuntimeOptions(ociRuntime, c.config)
if err != nil {
return nil, errors.Wrap(err, "failed to generate runtime options")
}
opts := []containerd.NewContainerOpts{
containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
customopts.WithNewSnapshot(id, image.Image),
containerd.WithSpec(spec, specOpts...),
containerd.WithContainerLabels(sandboxLabels),
containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata),
containerd.WithRuntime(
ociRuntime.Type,
&runctypes.RuncOptions{
Runtime: ociRuntime.Engine,
RuntimeRoot: ociRuntime.Root,
SystemdCgroup: c.config.SystemdCgroup})} // TODO (mikebrow): add CriuPath when we add support for pause
containerd.WithRuntime(ociRuntime.Type, runtimeOpts)}
container, err := c.client.NewContainer(ctx, id, opts...)
if err != nil {
@@ -297,7 +295,8 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
id, name)
var taskOpts []containerd.NewTaskOpts
if c.config.NoPivot {
// TODO(random-liu): Remove this after shim v1 is deprecated.
if c.config.NoPivot && ociRuntime.Type == linuxRuntime {
taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
}
// We don't need stdio for sandbox container.

11
pkg/server/sandbox_status.go
View File

@@ -26,7 +26,6 @@ import (
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
)
@@ -109,7 +108,8 @@ type sandboxInfo struct {
SnapshotKey string `json:"snapshotKey"`
Snapshotter string `json:"snapshotter"`
RuntimeHandler string `json:"runtimeHandler"`
Runtime *criconfig.Runtime `json:"runtime"`
RuntimeType string `json:"runtimeType"`
RuntimeOptions interface{} `json:"runtimeOptions"`
Config *runtime.PodSandboxConfig `json:"config"`
RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"`
}
@@ -167,11 +167,12 @@ func toCRISandboxInfo(ctx context.Context, sandbox sandboxstore.Sandbox) (map[st
si.SnapshotKey = ctrInfo.SnapshotKey
si.Snapshotter = ctrInfo.Snapshotter
ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo)
runtimeOptions, err := getRuntimeOptions(ctrInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to get sandbox container runtime config")
return nil, errors.Wrap(err, "failed to get runtime options")
}
si.Runtime = &ociRuntime
si.RuntimeType = ctrInfo.Runtime.Name
si.RuntimeOptions = runtimeOptions
infoBytes, err := json.Marshal(si)
if err != nil {