cri: implement CDI device injection
Extract the names of requested CDI devices and update the OCI Spec according to the corresponding CDI device specifications. CDI devices are requested using container annotations in the cdi.k8s.io namespace. Once CRI gains dedicated fields for CDI injection the snippet for extracting CDI names will need an update. Signed-off-by: Ed Bartosh <eduard.bartosh@intel.com>
This commit is contained in:
Ed Bartosh
parent
ffddd4446c
commit
aed0538dac
@ -22,10 +22,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
|
||||
"github.com/containerd/containerd/containers"
|
||||
"github.com/containerd/containerd/content"
|
||||
"github.com/containerd/containerd/errdefs"
|
||||
"github.com/containerd/containerd/images"
|
||||
"github.com/containerd/containerd/log"
|
||||
"github.com/containerd/containerd/oci"
|
||||
"github.com/containerd/containerd/protobuf"
|
||||
"github.com/containerd/containerd/snapshots"
|
||||
@ -324,3 +326,37 @@ func WithSpec(s *oci.Spec, opts ...oci.SpecOpts) NewContainerOpts {
|
||||
func WithoutRefreshedMetadata(i *InfoConfig) {
|
||||
i.Refresh = false
|
||||
}
|
||||
|
||||
// WithCDI updates OCI spec with CDI content
|
||||
func WithCDI(s *oci.Spec, annotations map[string]string) NewContainerOpts {
|
||||
return func(ctx context.Context, _ *Client, c *containers.Container) error {
|
||||
// TODO: Once CRI is extended with native CDI support this will need to be updated...
|
||||
_, cdiDevices, err := cdi.ParseAnnotations(annotations)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse CDI device annotations: %w", err)
|
||||
}
|
||||
if cdiDevices == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
registry := cdi.GetRegistry()
|
||||
if err = registry.Refresh(); err != nil {
|
||||
// We don't consider registry refresh failure a fatal error.
|
||||
// For instance, a dynamically generated invalid CDI Spec file for
|
||||
// any particular vendor shouldn't prevent injection of devices of
|
||||
// different vendors. CDI itself knows better and it will fail the
|
||||
// injection if necessary.
|
||||
log.G(ctx).Warnf("CDI registry refresh failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := registry.InjectDevices(s, cdiDevices...); err != nil {
|
||||
return fmt.Errorf("CDI device injection failed: %w", err)
|
||||
}
|
||||
|
||||
// One crucial thing to keep in mind is that CDI device injection
|
||||
// might add OCI Spec environment variables, hooks, and mounts as
|
||||
// well. Therefore it is important that none of the corresponding
|
||||
// OCI Spec fields are reset up in the call stack once we return.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@ -239,6 +239,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
|
||||
return nil, fmt.Errorf("failed to get runtime options: %w", err)
|
||||
}
|
||||
opts = append(opts,
|
||||
containerd.WithCDI(spec, config.Annotations),
|
||||
containerd.WithSpec(spec, specOpts...),
|
||||
containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions),
|
||||
containerd.WithContainerLabels(containerLabels),
|
||||
|
||||
@ -20,12 +20,15 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
|
||||
"github.com/containerd/containerd"
|
||||
"github.com/containerd/containerd/containers"
|
||||
"github.com/containerd/containerd/contrib/apparmor"
|
||||
"github.com/containerd/containerd/contrib/seccomp"
|
||||
@ -1485,3 +1488,152 @@ func TestBaseOCISpec(t *testing.T) {
|
||||
|
||||
assert.Equal(t, *spec.Linux.Resources.Memory.Limit, containerConfig.Linux.Resources.MemoryLimitInBytes)
|
||||
}
|
||||
|
||||
func writeFilesToTempDir(tmpDirPattern string, content []string) (string, error) {
|
||||
if len(content) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
dir, err := ioutil.TempDir("", tmpDirPattern)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for idx, data := range content {
|
||||
file := filepath.Join(dir, fmt.Sprintf("spec-%d.yaml", idx))
|
||||
err := ioutil.WriteFile(file, []byte(data), 0644)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return dir, cdi.GetRegistry(cdi.WithSpecDirs(dir)).Refresh()
|
||||
}
|
||||
|
||||
func TestCDIInjections(t *testing.T) {
|
||||
testID := "test-id"
|
||||
testSandboxID := "sandbox-id"
|
||||
testContainerName := "container-name"
|
||||
testPid := uint32(1234)
|
||||
containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
|
||||
ociRuntime := config.Runtime{}
|
||||
c := newTestCRIService()
|
||||
|
||||
for _, test := range []struct {
|
||||
description string
|
||||
cdiSpecFiles []string
|
||||
annotations map[string]string
|
||||
expectError bool
|
||||
expectDevices []runtimespec.LinuxDevice
|
||||
expectEnv []string
|
||||
}{
|
||||
{description: "expect no CDI error for nil annotations"},
|
||||
{description: "expect no CDI error for empty annotations",
|
||||
annotations: map[string]string{},
|
||||
},
|
||||
{description: "expect CDI error for invalid CDI device reference in annotations",
|
||||
annotations: map[string]string{
|
||||
cdi.AnnotationPrefix + "devices": "foobar",
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
{description: "expect CDI error for unresolvable devices",
|
||||
annotations: map[string]string{
|
||||
cdi.AnnotationPrefix + "vendor1_devices": "vendor1.com/device=no-such-dev",
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
{description: "expect properly injected resolvable CDI devices",
|
||||
cdiSpecFiles: []string{
|
||||
`
|
||||
cdiVersion: "0.2.0"
|
||||
kind: "vendor1.com/device"
|
||||
devices:
|
||||
- name: foo
|
||||
containerEdits:
|
||||
deviceNodes:
|
||||
- path: /dev/loop8
|
||||
type: b
|
||||
major: 7
|
||||
minor: 8
|
||||
env:
|
||||
- FOO=injected
|
||||
containerEdits:
|
||||
env:
|
||||
- "VENDOR1=present"
|
||||
`,
|
||||
`
|
||||
cdiVersion: "0.2.0"
|
||||
kind: "vendor2.com/device"
|
||||
devices:
|
||||
- name: bar
|
||||
containerEdits:
|
||||
deviceNodes:
|
||||
- path: /dev/loop9
|
||||
type: b
|
||||
major: 7
|
||||
minor: 9
|
||||
env:
|
||||
- BAR=injected
|
||||
containerEdits:
|
||||
env:
|
||||
- "VENDOR2=present"
|
||||
`,
|
||||
},
|
||||
annotations: map[string]string{
|
||||
cdi.AnnotationPrefix + "vendor1_devices": "vendor1.com/device=foo",
|
||||
cdi.AnnotationPrefix + "vendor2_devices": "vendor2.com/device=bar",
|
||||
},
|
||||
expectDevices: []runtimespec.LinuxDevice{
|
||||
{
|
||||
Path: "/dev/loop8",
|
||||
Type: "b",
|
||||
Major: 7,
|
||||
Minor: 8,
|
||||
},
|
||||
{
|
||||
Path: "/dev/loop9",
|
||||
Type: "b",
|
||||
Major: 7,
|
||||
Minor: 9,
|
||||
},
|
||||
},
|
||||
expectEnv: []string{
|
||||
"FOO=injected",
|
||||
"VENDOR1=present",
|
||||
"BAR=injected",
|
||||
"VENDOR2=present",
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Logf("TestCase %q", test.description)
|
||||
|
||||
spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
|
||||
require.NoError(t, err)
|
||||
|
||||
specCheck(t, testID, testSandboxID, testPid, spec)
|
||||
|
||||
cdiDir, err := writeFilesToTempDir("containerd-test-CDI-injections-", test.cdiSpecFiles)
|
||||
if cdiDir != "" {
|
||||
defer os.RemoveAll(cdiDir)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
injectFun := containerd.WithCDI(spec, test.annotations)
|
||||
err = injectFun(nil, nil, nil)
|
||||
assert.Equal(t, test.expectError, err != nil)
|
||||
|
||||
if err != nil {
|
||||
if test.expectEnv != nil {
|
||||
for _, expectedEnv := range test.expectEnv {
|
||||
assert.Contains(t, spec.Process.Env, expectedEnv)
|
||||
}
|
||||
}
|
||||
if test.expectDevices != nil {
|
||||
for _, expectedDev := range test.expectDevices {
|
||||
assert.Contains(t, spec.Linux.Devices, expectedDev)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user