github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Support custom runtime path when launching tasks

Browse Source 
Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>
This commit is contained in:
Maksym Pavlenko
2021-11-09 13:31:46 -08:00
parent d4f4c1380a
commit 6870f3b1b8
10 changed files with 146 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
runtime/runtime.go
View File

@@ -46,7 +46,8 @@ type CreateOpts struct {
RuntimeOptions *types.Any
// TaskOptions received for the task
TaskOptions *types.Any
// Runtime to use
// Runtime name to use (e.g. `io.containerd.NAME.VERSION`).
// As an alternative full abs path to binary may be specified instead.
Runtime string
}

5
runtime/v2/binary.go
View File

@@ -21,6 +21,7 @@ import (
"context"
"io"
"os"
"path/filepath"
gruntime "runtime"
"strings"
@@ -127,6 +128,10 @@ func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_
cancelShimLog()
f.Close()
}
// Save runtime binary path for restore.
if err := os.WriteFile(filepath.Join(b.bundle.Path, "runtime"), []byte(b.runtime), 0600); err != nil {
return nil, err
}
client := ttrpc.NewClient(conn, ttrpc.WithOnClose(onCloseWithShimLog))
return &shim{
bundle: b.bundle,

86
runtime/v2/manager.go
View File

@@ -20,6 +20,9 @@ import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"sync"
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/errdefs"
@@ -31,6 +34,7 @@ import (
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/runtime"
shim_binary "github.com/containerd/containerd/runtime/v2/shim"
"github.com/containerd/containerd/runtime/v2/task"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@@ -199,6 +203,8 @@ type ShimManager struct {
shims *runtime.TaskList
events *exchange.Exchange
containers containers.Store
// runtimePaths is a cache of `runtime names` -> `resolved fs path`
runtimePaths sync.Map
}
// ID of the shim manager
@@ -253,8 +259,13 @@ func (m *ShimManager) startShim(ctx context.Context, bundle *Bundle, id string,
topts = opts.RuntimeOptions
}
runtimePath, err := m.resolveRuntimePath(opts.Runtime)
if err != nil {
return nil, fmt.Errorf("failed to resolve runtime path: %w", err)
}
b := shimBinary(bundle, shimBinaryConfig{
runtime: opts.Runtime,
runtime: runtimePath,
address: m.containerdAddress,
ttrpcAddress: m.containerdTTRPCAddress,
schedCore: m.schedCore,
@@ -276,6 +287,79 @@ func (m *ShimManager) startShim(ctx context.Context, bundle *Bundle, id string,
return shim, nil
}
func (m *ShimManager) resolveRuntimePath(runtime string) (string, error) {
if runtime == "" {
return "", fmt.Errorf("no runtime name")
}
// Custom path to runtime binary
if filepath.IsAbs(runtime) {
// Make sure it exists before returning ok
if _, err := os.Stat(runtime); err != nil {
return "", fmt.Errorf("invalid custom binary path: %w", err)
}
return runtime, nil
}
// Preserve existing logic and resolve runtime path from runtime name.
name := shim_binary.BinaryName(runtime)
if name == "" {
return "", fmt.Errorf("invalid runtime name %s, correct runtime name should be either format like `io.containerd.runc.v1` or a full path to the binary", runtime)
}
if path, ok := m.runtimePaths.Load(name); ok {
return path.(string), nil
}
var (
cmdPath string
lerr error
)
binaryPath := shim_binary.BinaryPath(runtime)
if _, serr := os.Stat(binaryPath); serr == nil {
cmdPath = binaryPath
}
if cmdPath == "" {
if cmdPath, lerr = exec.LookPath(name); lerr != nil {
if eerr, ok := lerr.(*exec.Error); ok {
if eerr.Err == exec.ErrNotFound {
self, err := os.Executable()
if err != nil {
return "", err
}
// Match the calling binaries (containerd) path and see
// if they are side by side. If so, execute the shim
// found there.
testPath := filepath.Join(filepath.Dir(self), name)
if _, serr := os.Stat(testPath); serr == nil {
cmdPath = testPath
}
if cmdPath == "" {
return "", errors.Wrapf(os.ErrNotExist, "runtime %q binary not installed %q", runtime, name)
}
}
}
}
}
cmdPath, err := filepath.Abs(cmdPath)
if err != nil {
return "", err
}
if path, ok := m.runtimePaths.LoadOrStore(name, cmdPath); ok {
// We didn't store cmdPath we loaded an already cached value. Use it.
cmdPath = path.(string)
}
return cmdPath, nil
}
// cleanupShim attempts to properly delete and cleanup shim after error
func (m *ShimManager) cleanupShim(shim *shim) {
dctx, cancel := timeout.WithContext(context.Background(), cleanupTimeout)

49
runtime/v2/shim/util.go
View File

@@ -24,7 +24,6 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/containerd/containerd/namespaces"
@@ -34,8 +33,6 @@ import (
exec "golang.org/x/sys/execabs"
)
var runtimePaths sync.Map
type CommandConfig struct {
Runtime string
Address string
@@ -62,51 +59,7 @@ func Command(ctx context.Context, config *CommandConfig) (*exec.Cmd, error) {
"-publish-binary", self,
}
args = append(args, config.Args...)
name := BinaryName(config.Runtime)
if name == "" {
return nil, fmt.Errorf("invalid runtime name %s, correct runtime name should format like io.containerd.runc.v1", config.Runtime)
}
var cmdPath string
cmdPathI, cmdPathFound := runtimePaths.Load(name)
if cmdPathFound {
cmdPath = cmdPathI.(string)
} else {
var lerr error
binaryPath := BinaryPath(config.Runtime)
if _, serr := os.Stat(binaryPath); serr == nil {
cmdPath = binaryPath
}
if cmdPath == "" {
if cmdPath, lerr = exec.LookPath(name); lerr != nil {
if eerr, ok := lerr.(*exec.Error); ok {
if eerr.Err == exec.ErrNotFound {
// Match the calling binaries (containerd) path and see
// if they are side by side. If so, execute the shim
// found there.
testPath := filepath.Join(filepath.Dir(self), name)
if _, serr := os.Stat(testPath); serr == nil {
cmdPath = testPath
}
if cmdPath == "" {
return nil, errors.Wrapf(os.ErrNotExist, "runtime %q binary not installed %q", config.Runtime, name)
}
}
}
}
}
cmdPath, err = filepath.Abs(cmdPath)
if err != nil {
return nil, err
}
if cmdPathI, cmdPathFound = runtimePaths.LoadOrStore(name, cmdPath); cmdPathFound {
// We didn't store cmdPath we loaded an already cached value. Use it.
cmdPath = cmdPathI.(string)
}
}
cmd := exec.CommandContext(ctx, cmdPath, args...)
cmd := exec.CommandContext(ctx, config.Runtime, args...)
cmd.Dir = config.Path
cmd.Env = append(
os.Environ(),

35
runtime/v2/shim_load.go
View File

@@ -88,18 +88,35 @@ func (m *ShimManager) loadShims(ctx context.Context) error {
bundle.Delete()
continue
}
container, err := m.containers.Get(ctx, id)
if err != nil {
log.G(ctx).WithError(err).Errorf("loading container %s", id)
if err := mount.UnmountAll(filepath.Join(bundle.Path, "rootfs"), 0); err != nil {
log.G(ctx).WithError(err).Errorf("forceful unmount of rootfs %s", id)
}
bundle.Delete()
continue
var (
runtime string
)
// If we're on 1.6+ and specified custom path to the runtime binary, path will be saved in 'runtime' file.
if data, err := os.ReadFile(filepath.Join(bundle.Path, "runtime")); err == nil {
runtime = string(data)
} else if err != nil && !os.IsNotExist(err) {
log.G(ctx).WithError(err).Error("failed to read `runtime` path from bundle")
}
// Query runtime name from metadata store
if runtime == "" {
container, err := m.containers.Get(ctx, id)
if err != nil {
log.G(ctx).WithError(err).Errorf("loading container %s", id)
if err := mount.UnmountAll(filepath.Join(bundle.Path, "rootfs"), 0); err != nil {
log.G(ctx).WithError(err).Errorf("failed to unmount of rootfs %s", id)
}
bundle.Delete()
continue
}
runtime = container.Runtime.Name
}
binaryCall := shimBinary(bundle,
shimBinaryConfig{
runtime: container.Runtime.Name,
runtime: runtime,
address: m.containerdAddress,
ttrpcAddress: m.containerdTTRPCAddress,
schedCore: m.schedCore,