diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go index 1259de596..90b1520fd 100644 --- a/cmd/containerd-shim/main_unix.go +++ b/cmd/containerd-shim/main_unix.go @@ -58,6 +58,10 @@ func main() { Name: "address,a", Usage: "grpc address back to containerd", }, + cli.StringFlag{ + Name: "workdir,w", + Usage: "path used to store large temporary data", + }, } app.Before = func(context *cli.Context) error { if context.GlobalBool("debug") { @@ -84,6 +88,7 @@ func main() { sv, err := shim.NewService( path, context.GlobalString("namespace"), + context.GlobalString("workdir"), &remoteEventsPublisher{client: e}, ) if err != nil { diff --git a/linux/bundle.go b/linux/bundle.go index 49615002f..9204fc74e 100644 --- a/linux/bundle.go +++ b/linux/bundle.go @@ -25,7 +25,7 @@ func loadBundle(path, namespace string, events *events.Exchange) *bundle { } // newBundle creates a new bundle on disk at the provided path for the given id -func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) (b *bundle, err error) { +func newBundle(path, namespace, workDir, id string, spec []byte, events *events.Exchange) (b *bundle, err error) { if err := os.MkdirAll(path, 0711); err != nil { return nil, err } @@ -35,6 +35,16 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) os.RemoveAll(path) } }() + workDir = filepath.Join(workDir, id) + if err := os.MkdirAll(workDir, 0711); err != nil { + return nil, err + } + defer func() { + if err != nil { + os.RemoveAll(workDir) + } + }() + if err := os.Mkdir(path, 0711); err != nil { return nil, err } @@ -50,6 +60,7 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) return &bundle{ id: id, path: path, + workDir: workDir, namespace: namespace, events: events, }, err @@ -58,6 +69,7 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) type bundle struct { id string path string + workDir string namespace string events *events.Exchange } @@ -81,6 +93,7 @@ func (b *bundle) NewShim(ctx context.Context, binary, grpcAddress string, remote Path: b.path, Namespace: b.namespace, CgroupPath: options.ShimCgroup, + WorkDir: b.workDir, }, opt) } diff --git a/linux/runtime.go b/linux/runtime.go index 712175210..7f54ff1d0 100644 --- a/linux/runtime.go +++ b/linux/runtime.go @@ -69,6 +69,9 @@ type Config struct { } func New(ic *plugin.InitContext) (interface{}, error) { + if err := os.MkdirAll(ic.Root, 0711); err != nil { + return nil, err + } if err := os.MkdirAll(ic.State, 0711); err != nil { return nil, err } @@ -82,7 +85,8 @@ func New(ic *plugin.InitContext) (interface{}, error) { } cfg := ic.Config.(*Config) r := &Runtime{ - root: ic.State, + root: ic.Root, + state: ic.State, remote: !cfg.NoShim, shim: cfg.Shim, shimDebug: cfg.ShimDebug, @@ -107,6 +111,7 @@ func New(ic *plugin.InitContext) (interface{}, error) { type Runtime struct { root string + state string shim string shimDebug bool runtime string @@ -133,7 +138,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts return nil, errors.Wrapf(err, "invalid task id") } - bundle, err := newBundle(filepath.Join(r.root, namespace), namespace, id, opts.Spec.Value, r.events) + bundle, err := newBundle(filepath.Join(r.state, namespace), namespace, filepath.Join(r.root, namespace), id, opts.Spec.Value, r.events) if err != nil { return nil, err } @@ -206,7 +211,7 @@ func (r *Runtime) Delete(ctx context.Context, c runtime.Task) (*runtime.Exit, er } r.tasks.Delete(ctx, lc) - bundle := loadBundle(filepath.Join(r.root, namespace, lc.id), namespace, r.events) + bundle := loadBundle(filepath.Join(r.state, namespace, lc.id), namespace, r.events) if err := bundle.Delete(); err != nil { return nil, err } @@ -222,7 +227,7 @@ func (r *Runtime) Tasks(ctx context.Context) ([]runtime.Task, error) { } func (r *Runtime) restoreTasks(ctx context.Context) ([]*Task, error) { - dir, err := ioutil.ReadDir(r.root) + dir, err := ioutil.ReadDir(r.state) if err != nil { return nil, err } @@ -247,7 +252,7 @@ func (r *Runtime) Get(ctx context.Context, id string) (runtime.Task, error) { } func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { - dir, err := ioutil.ReadDir(filepath.Join(r.root, ns)) + dir, err := ioutil.ReadDir(filepath.Join(r.state, ns)) if err != nil { return nil, err } @@ -257,7 +262,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { continue } id := path.Name() - bundle := loadBundle(filepath.Join(r.root, ns, id), ns, r.events) + bundle := loadBundle(filepath.Join(r.state, ns, id), ns, r.events) s, err := bundle.Connect(ctx, r.remote) if err != nil { diff --git a/linux/shim/client.go b/linux/shim/client.go index e80cb239d..ea634a8f8 100644 --- a/linux/shim/client.go +++ b/linux/shim/client.go @@ -81,6 +81,7 @@ func newCommand(binary, address string, debug bool, config Config, socket *os.Fi args := []string{ "--namespace", config.Namespace, "--address", address, + "--workdir", config.WorkDir, } if debug { args = append(args, "--debug") @@ -152,7 +153,7 @@ func WithConnect(ctx context.Context, config Config) (shim.ShimClient, io.Closer // WithLocal uses an in process shim func WithLocal(publisher events.Publisher) func(context.Context, Config) (shim.ShimClient, io.Closer, error) { return func(ctx context.Context, config Config) (shim.ShimClient, io.Closer, error) { - service, err := NewService(config.Path, config.Namespace, publisher) + service, err := NewService(config.Path, config.Namespace, config.WorkDir, publisher) if err != nil { return nil, nil, err } @@ -165,6 +166,7 @@ type Config struct { Path string Namespace string CgroupPath string + WorkDir string } // New returns a new shim client diff --git a/linux/shim/init.go b/linux/shim/init.go index 66fb55152..aad6475b2 100644 --- a/linux/shim/init.go +++ b/linux/shim/init.go @@ -39,6 +39,8 @@ type initProcess struct { // the reaper interface. mu sync.Mutex + workDir string + id string bundle string console console.Console @@ -54,7 +56,7 @@ type initProcess struct { rootfs string } -func newInitProcess(context context.Context, plat platform, path, namespace string, r *shimapi.CreateTaskRequest) (*initProcess, error) { +func newInitProcess(context context.Context, plat platform, path, namespace, workDir string, r *shimapi.CreateTaskRequest) (*initProcess, error) { var success bool if err := identifiers.Validate(r.ID); err != nil { @@ -109,7 +111,8 @@ func newInitProcess(context context.Context, plat platform, path, namespace stri stderr: r.Stderr, terminal: r.Terminal, }, - rootfs: rootfs, + rootfs: rootfs, + workDir: workDir, } var ( err error @@ -132,7 +135,7 @@ func newInitProcess(context context.Context, plat platform, path, namespace stri opts := &runc.RestoreOpts{ CheckpointOpts: runc.CheckpointOpts{ ImagePath: r.Checkpoint, - WorkDir: filepath.Join(r.Bundle, "work"), + WorkDir: p.workDir, ParentPath: r.ParentCheckpoint, }, PidFile: pidFile, @@ -310,10 +313,10 @@ func (p *initProcess) Checkpoint(context context.Context, r *shimapi.CheckpointT if !options.Exit { actions = append(actions, runc.LeaveRunning) } - work := filepath.Join(p.bundle, "work") + work := filepath.Join(p.workDir, "criu-work") defer os.RemoveAll(work) if err := p.runtime.Checkpoint(context, p.id, &runc.CheckpointOpts{ - WorkDir: work, + WorkDir: p.workDir, ImagePath: r.Path, AllowOpenTCP: options.OpenTcp, AllowExternalUnixSockets: options.ExternalUnixSockets, diff --git a/linux/shim/service.go b/linux/shim/service.go index 5957351c4..1bdf61390 100644 --- a/linux/shim/service.go +++ b/linux/shim/service.go @@ -30,7 +30,7 @@ var empty = &google_protobuf.Empty{} const RuncRoot = "/run/containerd/runc" // NewService returns a new shim service that can be used via GRPC -func NewService(path, namespace string, publisher events.Publisher) (*Service, error) { +func NewService(path, namespace, workDir string, publisher events.Publisher) (*Service, error) { if namespace == "" { return nil, fmt.Errorf("shim namespace cannot be empty") } @@ -41,6 +41,7 @@ func NewService(path, namespace string, publisher events.Publisher) (*Service, e events: make(chan interface{}, 4096), namespace: namespace, context: context, + workDir: workDir, } if err := s.initPlatform(); err != nil { return nil, errors.Wrap(err, "failed to initialized platform behavior") @@ -69,11 +70,12 @@ type Service struct { namespace string context context.Context + workDir string platform platform } func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (*shimapi.CreateTaskResponse, error) { - process, err := newInitProcess(ctx, s.platform, s.path, s.namespace, r) + process, err := newInitProcess(ctx, s.platform, s.path, s.namespace, s.workDir, r) if err != nil { return nil, errdefs.ToGRPC(err) }