diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go index 1259de596..90b1520fd 100644 --- a/cmd/containerd-shim/main_unix.go +++ b/cmd/containerd-shim/main_unix.go @@ -58,6 +58,10 @@ func main() { Name: "address,a", Usage: "grpc address back to containerd", }, + cli.StringFlag{ + Name: "workdir,w", + Usage: "path used to store large temporary data", + }, } app.Before = func(context *cli.Context) error { if context.GlobalBool("debug") { @@ -84,6 +88,7 @@ func main() { sv, err := shim.NewService( path, context.GlobalString("namespace"), + context.GlobalString("workdir"), &remoteEventsPublisher{client: e}, ) if err != nil { diff --git a/cmd/containerd/config_linux.go b/cmd/containerd/config_linux.go index 538c37bf9..68d3f1629 100644 --- a/cmd/containerd/config_linux.go +++ b/cmd/containerd/config_linux.go @@ -6,7 +6,8 @@ import ( func defaultConfig() *server.Config { return &server.Config{ - Root: "/var/lib/containerd", + Root: server.DefaultRootDir, + State: server.DefaultStateDir, GRPC: server.GRPCConfig{ Address: server.DefaultAddress, }, diff --git a/cmd/containerd/config_unsupported.go b/cmd/containerd/config_unsupported.go index 6a7478f44..8f52def85 100644 --- a/cmd/containerd/config_unsupported.go +++ b/cmd/containerd/config_unsupported.go @@ -6,7 +6,8 @@ import "github.com/containerd/containerd/server" func defaultConfig() *server.Config { return &server.Config{ - Root: "/var/lib/containerd", + Root: server.DefaultRootDir, + State: server.DefaultStateDir, GRPC: server.GRPCConfig{ Address: server.DefaultAddress, }, diff --git a/cmd/containerd/config_windows.go b/cmd/containerd/config_windows.go index 20194221f..c11145172 100644 --- a/cmd/containerd/config_windows.go +++ b/cmd/containerd/config_windows.go @@ -1,15 +1,11 @@ package main -import ( - "os" - "path/filepath" - - "github.com/containerd/containerd/server" -) +import "github.com/containerd/containerd/server" func defaultConfig() *server.Config { return &server.Config{ - Root: filepath.Join(os.Getenv("programfiles"), "containerd", "root"), + Root: server.DefaultRootDir, + State: server.DefaultStateDir, GRPC: server.GRPCConfig{ Address: server.DefaultAddress, }, diff --git a/linux/bundle.go b/linux/bundle.go index 49615002f..9204fc74e 100644 --- a/linux/bundle.go +++ b/linux/bundle.go @@ -25,7 +25,7 @@ func loadBundle(path, namespace string, events *events.Exchange) *bundle { } // newBundle creates a new bundle on disk at the provided path for the given id -func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) (b *bundle, err error) { +func newBundle(path, namespace, workDir, id string, spec []byte, events *events.Exchange) (b *bundle, err error) { if err := os.MkdirAll(path, 0711); err != nil { return nil, err } @@ -35,6 +35,16 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) os.RemoveAll(path) } }() + workDir = filepath.Join(workDir, id) + if err := os.MkdirAll(workDir, 0711); err != nil { + return nil, err + } + defer func() { + if err != nil { + os.RemoveAll(workDir) + } + }() + if err := os.Mkdir(path, 0711); err != nil { return nil, err } @@ -50,6 +60,7 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) return &bundle{ id: id, path: path, + workDir: workDir, namespace: namespace, events: events, }, err @@ -58,6 +69,7 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) type bundle struct { id string path string + workDir string namespace string events *events.Exchange } @@ -81,6 +93,7 @@ func (b *bundle) NewShim(ctx context.Context, binary, grpcAddress string, remote Path: b.path, Namespace: b.namespace, CgroupPath: options.ShimCgroup, + WorkDir: b.workDir, }, opt) } diff --git a/linux/runtime.go b/linux/runtime.go index 049c4328d..7f54ff1d0 100644 --- a/linux/runtime.go +++ b/linux/runtime.go @@ -72,6 +72,9 @@ func New(ic *plugin.InitContext) (interface{}, error) { if err := os.MkdirAll(ic.Root, 0711); err != nil { return nil, err } + if err := os.MkdirAll(ic.State, 0711); err != nil { + return nil, err + } monitor, err := ic.Get(plugin.TaskMonitorPlugin) if err != nil { return nil, err @@ -83,6 +86,7 @@ func New(ic *plugin.InitContext) (interface{}, error) { cfg := ic.Config.(*Config) r := &Runtime{ root: ic.Root, + state: ic.State, remote: !cfg.NoShim, shim: cfg.Shim, shimDebug: cfg.ShimDebug, @@ -107,6 +111,7 @@ func New(ic *plugin.InitContext) (interface{}, error) { type Runtime struct { root string + state string shim string shimDebug bool runtime string @@ -133,7 +138,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts return nil, errors.Wrapf(err, "invalid task id") } - bundle, err := newBundle(filepath.Join(r.root, namespace), namespace, id, opts.Spec.Value, r.events) + bundle, err := newBundle(filepath.Join(r.state, namespace), namespace, filepath.Join(r.root, namespace), id, opts.Spec.Value, r.events) if err != nil { return nil, err } @@ -206,7 +211,7 @@ func (r *Runtime) Delete(ctx context.Context, c runtime.Task) (*runtime.Exit, er } r.tasks.Delete(ctx, lc) - bundle := loadBundle(filepath.Join(r.root, namespace, lc.id), namespace, r.events) + bundle := loadBundle(filepath.Join(r.state, namespace, lc.id), namespace, r.events) if err := bundle.Delete(); err != nil { return nil, err } @@ -222,7 +227,7 @@ func (r *Runtime) Tasks(ctx context.Context) ([]runtime.Task, error) { } func (r *Runtime) restoreTasks(ctx context.Context) ([]*Task, error) { - dir, err := ioutil.ReadDir(r.root) + dir, err := ioutil.ReadDir(r.state) if err != nil { return nil, err } @@ -247,7 +252,7 @@ func (r *Runtime) Get(ctx context.Context, id string) (runtime.Task, error) { } func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { - dir, err := ioutil.ReadDir(filepath.Join(r.root, ns)) + dir, err := ioutil.ReadDir(filepath.Join(r.state, ns)) if err != nil { return nil, err } @@ -257,7 +262,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { continue } id := path.Name() - bundle := loadBundle(filepath.Join(r.root, ns, id), ns, r.events) + bundle := loadBundle(filepath.Join(r.state, ns, id), ns, r.events) s, err := bundle.Connect(ctx, r.remote) if err != nil { diff --git a/linux/shim/client.go b/linux/shim/client.go index e80cb239d..ea634a8f8 100644 --- a/linux/shim/client.go +++ b/linux/shim/client.go @@ -81,6 +81,7 @@ func newCommand(binary, address string, debug bool, config Config, socket *os.Fi args := []string{ "--namespace", config.Namespace, "--address", address, + "--workdir", config.WorkDir, } if debug { args = append(args, "--debug") @@ -152,7 +153,7 @@ func WithConnect(ctx context.Context, config Config) (shim.ShimClient, io.Closer // WithLocal uses an in process shim func WithLocal(publisher events.Publisher) func(context.Context, Config) (shim.ShimClient, io.Closer, error) { return func(ctx context.Context, config Config) (shim.ShimClient, io.Closer, error) { - service, err := NewService(config.Path, config.Namespace, publisher) + service, err := NewService(config.Path, config.Namespace, config.WorkDir, publisher) if err != nil { return nil, nil, err } @@ -165,6 +166,7 @@ type Config struct { Path string Namespace string CgroupPath string + WorkDir string } // New returns a new shim client diff --git a/linux/shim/init.go b/linux/shim/init.go index acbc2e76a..4f4f42933 100644 --- a/linux/shim/init.go +++ b/linux/shim/init.go @@ -39,6 +39,8 @@ type initProcess struct { // the reaper interface. mu sync.Mutex + workDir string + id string bundle string console console.Console @@ -54,7 +56,7 @@ type initProcess struct { rootfs string } -func newInitProcess(context context.Context, plat platform, path, namespace string, r *shimapi.CreateTaskRequest) (*initProcess, error) { +func newInitProcess(context context.Context, plat platform, path, namespace, workDir string, r *shimapi.CreateTaskRequest) (*initProcess, error) { var success bool if err := identifiers.Validate(r.ID); err != nil { @@ -109,7 +111,8 @@ func newInitProcess(context context.Context, plat platform, path, namespace stri stderr: r.Stderr, terminal: r.Terminal, }, - rootfs: rootfs, + rootfs: rootfs, + workDir: workDir, } var ( err error @@ -132,7 +135,7 @@ func newInitProcess(context context.Context, plat platform, path, namespace stri opts := &runc.RestoreOpts{ CheckpointOpts: runc.CheckpointOpts{ ImagePath: r.Checkpoint, - WorkDir: filepath.Join(r.Bundle, "work"), + WorkDir: p.workDir, ParentPath: r.ParentCheckpoint, }, PidFile: pidFile, @@ -310,10 +313,10 @@ func (p *initProcess) Checkpoint(context context.Context, r *shimapi.CheckpointT if !options.Exit { actions = append(actions, runc.LeaveRunning) } - work := filepath.Join(p.bundle, "work") + work := filepath.Join(p.workDir, "criu-work") defer os.RemoveAll(work) if err := p.runtime.Checkpoint(context, p.id, &runc.CheckpointOpts{ - WorkDir: work, + WorkDir: p.workDir, ImagePath: r.Path, AllowOpenTCP: options.OpenTcp, AllowExternalUnixSockets: options.ExternalUnixSockets, diff --git a/linux/shim/service.go b/linux/shim/service.go index 17d228e51..be54104dc 100644 --- a/linux/shim/service.go +++ b/linux/shim/service.go @@ -30,7 +30,7 @@ var empty = &google_protobuf.Empty{} const RuncRoot = "/run/containerd/runc" // NewService returns a new shim service that can be used via GRPC -func NewService(path, namespace string, publisher events.Publisher) (*Service, error) { +func NewService(path, namespace, workDir string, publisher events.Publisher) (*Service, error) { if namespace == "" { return nil, fmt.Errorf("shim namespace cannot be empty") } @@ -41,6 +41,7 @@ func NewService(path, namespace string, publisher events.Publisher) (*Service, e events: make(chan interface{}, 4096), namespace: namespace, context: context, + workDir: workDir, } if err := s.initPlatform(); err != nil { return nil, errors.Wrap(err, "failed to initialized platform behavior") @@ -69,11 +70,12 @@ type Service struct { namespace string context context.Context + workDir string platform platform } func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (*shimapi.CreateTaskResponse, error) { - process, err := newInitProcess(ctx, s.platform, s.path, s.namespace, r) + process, err := newInitProcess(ctx, s.platform, s.path, s.namespace, s.workDir, r) if err != nil { return nil, errdefs.ToGRPC(err) } diff --git a/plugin/context.go b/plugin/context.go index 16cf1b8b7..dc5fb1528 100644 --- a/plugin/context.go +++ b/plugin/context.go @@ -9,16 +9,18 @@ import ( "github.com/containerd/containerd/log" ) -func NewContext(ctx context.Context, plugins map[PluginType]map[string]interface{}, root, id string) *InitContext { +func NewContext(ctx context.Context, plugins map[PluginType]map[string]interface{}, root, state, id string) *InitContext { return &InitContext{ plugins: plugins, Root: filepath.Join(root, id), + State: filepath.Join(state, id), Context: log.WithModule(ctx, id), } } type InitContext struct { Root string + State string Address string Context context.Context Config interface{} diff --git a/server/config.go b/server/config.go index 02659b669..dec0cfc57 100644 --- a/server/config.go +++ b/server/config.go @@ -11,6 +11,8 @@ import ( type Config struct { // Root is the path to a directory where containerd will store persistent data Root string `toml:"root"` + // State is the path to a directory where containerd will store transient data + State string `toml:"state"` // GRPC configuration settings GRPC GRPCConfig `toml:"grpc"` // Debug and profiling settings diff --git a/server/server.go b/server/server.go index 940f40eaf..40b140159 100644 --- a/server/server.go +++ b/server/server.go @@ -36,9 +36,15 @@ func New(ctx context.Context, config *Config) (*Server, error) { if config.Root == "" { return nil, errors.New("root must be specified") } + if config.State == "" { + return nil, errors.New("state must be specified") + } if err := os.MkdirAll(config.Root, 0711); err != nil { return nil, err } + if err := os.MkdirAll(config.State, 0711); err != nil { + return nil, err + } if err := apply(ctx, config); err != nil { return nil, err } @@ -66,6 +72,7 @@ func New(ctx context.Context, config *Config) (*Server, error) { ctx, initialized, config.Root, + config.State, id, ) initContext.Events = s.events diff --git a/server/server_linux.go b/server/server_linux.go index 58001cca0..34d35d012 100644 --- a/server/server_linux.go +++ b/server/server_linux.go @@ -9,6 +9,12 @@ import ( ) const ( + // DefaultRootDir is the default location used by containerd to store + // persistent data + DefaultRootDir = "/var/lib/containerd" + // DefaultStateDir is the default location used by containerd to store + // transient data + DefaultStateDir = "/run/containerd" // DefaultAddress is the default unix socket address DefaultAddress = "/run/containerd/containerd.sock" // DefaultDebuggAddress is the default unix socket address for pprof data diff --git a/server/server_unsupported.go b/server/server_unsupported.go index 197bfa029..0ec5d7e0a 100644 --- a/server/server_unsupported.go +++ b/server/server_unsupported.go @@ -5,6 +5,12 @@ package server import "context" const ( + // DefaultRootDir is the default location used by containerd to store + // persistent data + DefaultRootDir = "/var/lib/containerd" + // DefaultStateDir is the default location used by containerd to store + // transient data + DefaultStateDir = "/run/containerd" // DefaultAddress is the default unix socket address DefaultAddress = "/run/containerd/containerd.sock" // DefaultDebuggAddress is the default unix socket address for pprof data diff --git a/server/server_windows.go b/server/server_windows.go index 0fb9f64be..b35e776f7 100644 --- a/server/server_windows.go +++ b/server/server_windows.go @@ -2,7 +2,20 @@ package server -import "context" +import ( + "context" + "os" + "path/filepath" +) + +var ( + // DefaultRootDir is the default location used by containerd to store + // persistent data + DefaultRootDir = filepath.Join(os.Getenv("programfiles"), "containerd", "root") + // DefaultStateDir is the default location used by containerd to store + // transient data + DefaultStateDir = filepath.Join(os.Getenv("programfiles"), "containerd", "state") +) const ( // DefaultAddress is the default winpipe address