Merge pull request #1283 from mlaventure/resurrect-state-dir

Resurrect State directory

cmd/containerd-shim/main_unix.go

@@ -58,6 +58,10 @@ func main() {
 			Name:  "address,a",
 			Usage: "grpc address back to containerd",
 		},
+		cli.StringFlag{
+			Name:  "workdir,w",
+			Usage: "path used to store large temporary data",
+		},
 	}
 	app.Before = func(context *cli.Context) error {
 		if context.GlobalBool("debug") {
@@ -84,6 +88,7 @@ func main() {
 		sv, err := shim.NewService(
 			path,
 			context.GlobalString("namespace"),
+			context.GlobalString("workdir"),
 			&remoteEventsPublisher{client: e},
 		)
 		if err != nil {

cmd/containerd/config_linux.go

@@ -6,7 +6,8 @@ import (
 
 func defaultConfig() *server.Config {
 	return &server.Config{
-		Root: "/var/lib/containerd",
+		Root:  server.DefaultRootDir,
+		State: server.DefaultStateDir,
 		GRPC: server.GRPCConfig{
 			Address: server.DefaultAddress,
 		},

cmd/containerd/config_unsupported.go

@@ -6,7 +6,8 @@ import "github.com/containerd/containerd/server"
 
 func defaultConfig() *server.Config {
 	return &server.Config{
-		Root: "/var/lib/containerd",
+		Root:  server.DefaultRootDir,
+		State: server.DefaultStateDir,
 		GRPC: server.GRPCConfig{
 			Address: server.DefaultAddress,
 		},

cmd/containerd/config_windows.go

@@ -1,15 +1,11 @@
 package main
 
-import (
+import "github.com/containerd/containerd/server"
-	"os"
-	"path/filepath"
-
-	"github.com/containerd/containerd/server"
-)
 
 func defaultConfig() *server.Config {
 	return &server.Config{
-		Root: filepath.Join(os.Getenv("programfiles"), "containerd", "root"),
+		Root:  server.DefaultRootDir,
+		State: server.DefaultStateDir,
 		GRPC: server.GRPCConfig{
 			Address: server.DefaultAddress,
 		},

linux/bundle.go

@@ -25,7 +25,7 @@ func loadBundle(path, namespace string, events *events.Exchange) *bundle {
 }
 
 // newBundle creates a new bundle on disk at the provided path for the given id
-func newBundle(path, namespace, id string, spec []byte, events *events.Exchange) (b *bundle, err error) {
+func newBundle(path, namespace, workDir, id string, spec []byte, events *events.Exchange) (b *bundle, err error) {
 	if err := os.MkdirAll(path, 0711); err != nil {
 		return nil, err
 	}
@@ -35,6 +35,16 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange)
 			os.RemoveAll(path)
 		}
 	}()
+	workDir = filepath.Join(workDir, id)
+	if err := os.MkdirAll(workDir, 0711); err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			os.RemoveAll(workDir)
+		}
+	}()
+
 	if err := os.Mkdir(path, 0711); err != nil {
 		return nil, err
 	}
@@ -50,6 +60,7 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange)
 	return &bundle{
 		id:        id,
 		path:      path,
+		workDir:   workDir,
 		namespace: namespace,
 		events:    events,
 	}, err
@@ -58,6 +69,7 @@ func newBundle(path, namespace, id string, spec []byte, events *events.Exchange)
 type bundle struct {
 	id        string
 	path      string
+	workDir   string
 	namespace string
 	events    *events.Exchange
 }
@@ -81,6 +93,7 @@ func (b *bundle) NewShim(ctx context.Context, binary, grpcAddress string, remote
 		Path:       b.path,
 		Namespace:  b.namespace,
 		CgroupPath: options.ShimCgroup,
+		WorkDir:    b.workDir,
 	}, opt)
 }
 

linux/runtime.go

@@ -72,6 +72,9 @@ func New(ic *plugin.InitContext) (interface{}, error) {
 	if err := os.MkdirAll(ic.Root, 0711); err != nil {
 		return nil, err
 	}
+	if err := os.MkdirAll(ic.State, 0711); err != nil {
+		return nil, err
+	}
 	monitor, err := ic.Get(plugin.TaskMonitorPlugin)
 	if err != nil {
 		return nil, err
@@ -83,6 +86,7 @@ func New(ic *plugin.InitContext) (interface{}, error) {
 	cfg := ic.Config.(*Config)
 	r := &Runtime{
 		root:      ic.Root,
+		state:     ic.State,
 		remote:    !cfg.NoShim,
 		shim:      cfg.Shim,
 		shimDebug: cfg.ShimDebug,
@@ -107,6 +111,7 @@ func New(ic *plugin.InitContext) (interface{}, error) {
 
 type Runtime struct {
 	root      string
+	state     string
 	shim      string
 	shimDebug bool
 	runtime   string
@@ -133,7 +138,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
 		return nil, errors.Wrapf(err, "invalid task id")
 	}
 
-	bundle, err := newBundle(filepath.Join(r.root, namespace), namespace, id, opts.Spec.Value, r.events)
+	bundle, err := newBundle(filepath.Join(r.state, namespace), namespace, filepath.Join(r.root, namespace), id, opts.Spec.Value, r.events)
 	if err != nil {
 		return nil, err
 	}
@@ -206,7 +211,7 @@ func (r *Runtime) Delete(ctx context.Context, c runtime.Task) (*runtime.Exit, er
 	}
 	r.tasks.Delete(ctx, lc)
 
-	bundle := loadBundle(filepath.Join(r.root, namespace, lc.id), namespace, r.events)
+	bundle := loadBundle(filepath.Join(r.state, namespace, lc.id), namespace, r.events)
 	if err := bundle.Delete(); err != nil {
 		return nil, err
 	}
@@ -222,7 +227,7 @@ func (r *Runtime) Tasks(ctx context.Context) ([]runtime.Task, error) {
 }
 
 func (r *Runtime) restoreTasks(ctx context.Context) ([]*Task, error) {
-	dir, err := ioutil.ReadDir(r.root)
+	dir, err := ioutil.ReadDir(r.state)
 	if err != nil {
 		return nil, err
 	}
@@ -247,7 +252,7 @@ func (r *Runtime) Get(ctx context.Context, id string) (runtime.Task, error) {
 }
 
 func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
-	dir, err := ioutil.ReadDir(filepath.Join(r.root, ns))
+	dir, err := ioutil.ReadDir(filepath.Join(r.state, ns))
 	if err != nil {
 		return nil, err
 	}
@@ -257,7 +262,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
 			continue
 		}
 		id := path.Name()
-		bundle := loadBundle(filepath.Join(r.root, ns, id), ns, r.events)
+		bundle := loadBundle(filepath.Join(r.state, ns, id), ns, r.events)
 
 		s, err := bundle.Connect(ctx, r.remote)
 		if err != nil {

linux/shim/client.go

@@ -81,6 +81,7 @@ func newCommand(binary, address string, debug bool, config Config, socket *os.Fi
 	args := []string{
 		"--namespace", config.Namespace,
 		"--address", address,
+		"--workdir", config.WorkDir,
 	}
 	if debug {
 		args = append(args, "--debug")
@@ -152,7 +153,7 @@ func WithConnect(ctx context.Context, config Config) (shim.ShimClient, io.Closer
 // WithLocal uses an in process shim
 func WithLocal(publisher events.Publisher) func(context.Context, Config) (shim.ShimClient, io.Closer, error) {
 	return func(ctx context.Context, config Config) (shim.ShimClient, io.Closer, error) {
-		service, err := NewService(config.Path, config.Namespace, publisher)
+		service, err := NewService(config.Path, config.Namespace, config.WorkDir, publisher)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -165,6 +166,7 @@ type Config struct {
 	Path       string
 	Namespace  string
 	CgroupPath string
+	WorkDir    string
 }
 
 // New returns a new shim client

linux/shim/init.go

@@ -39,6 +39,8 @@ type initProcess struct {
 	// the reaper interface.
 	mu sync.Mutex
 
+	workDir string
+
 	id       string
 	bundle   string
 	console  console.Console
@@ -54,7 +56,7 @@ type initProcess struct {
 	rootfs   string
 }
 
-func newInitProcess(context context.Context, plat platform, path, namespace string, r *shimapi.CreateTaskRequest) (*initProcess, error) {
+func newInitProcess(context context.Context, plat platform, path, namespace, workDir string, r *shimapi.CreateTaskRequest) (*initProcess, error) {
 	var success bool
 
 	if err := identifiers.Validate(r.ID); err != nil {
@@ -109,7 +111,8 @@ func newInitProcess(context context.Context, plat platform, path, namespace stri
 			stderr:   r.Stderr,
 			terminal: r.Terminal,
 		},
-		rootfs: rootfs,
+		rootfs:  rootfs,
+		workDir: workDir,
 	}
 	var (
 		err    error
@@ -132,7 +135,7 @@ func newInitProcess(context context.Context, plat platform, path, namespace stri
 		opts := &runc.RestoreOpts{
 			CheckpointOpts: runc.CheckpointOpts{
 				ImagePath:  r.Checkpoint,
-				WorkDir:    filepath.Join(r.Bundle, "work"),
+				WorkDir:    p.workDir,
 				ParentPath: r.ParentCheckpoint,
 			},
 			PidFile:     pidFile,
@@ -310,10 +313,10 @@ func (p *initProcess) Checkpoint(context context.Context, r *shimapi.CheckpointT
 	if !options.Exit {
 		actions = append(actions, runc.LeaveRunning)
 	}
-	work := filepath.Join(p.bundle, "work")
+	work := filepath.Join(p.workDir, "criu-work")
 	defer os.RemoveAll(work)
 	if err := p.runtime.Checkpoint(context, p.id, &runc.CheckpointOpts{
-		WorkDir:                  work,
+		WorkDir:                  p.workDir,
 		ImagePath:                r.Path,
 		AllowOpenTCP:             options.OpenTcp,
 		AllowExternalUnixSockets: options.ExternalUnixSockets,

linux/shim/service.go

@@ -30,7 +30,7 @@ var empty = &google_protobuf.Empty{}
 const RuncRoot = "/run/containerd/runc"
 
 // NewService returns a new shim service that can be used via GRPC
-func NewService(path, namespace string, publisher events.Publisher) (*Service, error) {
+func NewService(path, namespace, workDir string, publisher events.Publisher) (*Service, error) {
 	if namespace == "" {
 		return nil, fmt.Errorf("shim namespace cannot be empty")
 	}
@@ -41,6 +41,7 @@ func NewService(path, namespace string, publisher events.Publisher) (*Service, e
 		events:    make(chan interface{}, 4096),
 		namespace: namespace,
 		context:   context,
+		workDir:   workDir,
 	}
 	if err := s.initPlatform(); err != nil {
 		return nil, errors.Wrap(err, "failed to initialized platform behavior")
@@ -69,11 +70,12 @@ type Service struct {
 	namespace     string
 	context       context.Context
 
+	workDir  string
 	platform platform
 }
 
 func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (*shimapi.CreateTaskResponse, error) {
-	process, err := newInitProcess(ctx, s.platform, s.path, s.namespace, r)
+	process, err := newInitProcess(ctx, s.platform, s.path, s.namespace, s.workDir, r)
 	if err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}

plugin/context.go

@@ -9,16 +9,18 @@ import (
 	"github.com/containerd/containerd/log"
 )
 
-func NewContext(ctx context.Context, plugins map[PluginType]map[string]interface{}, root, id string) *InitContext {
+func NewContext(ctx context.Context, plugins map[PluginType]map[string]interface{}, root, state, id string) *InitContext {
 	return &InitContext{
 		plugins: plugins,
 		Root:    filepath.Join(root, id),
+		State:   filepath.Join(state, id),
 		Context: log.WithModule(ctx, id),
 	}
 }
 
 type InitContext struct {
 	Root    string
+	State   string
 	Address string
 	Context context.Context
 	Config  interface{}

server/config.go

@@ -11,6 +11,8 @@ import (
 type Config struct {
 	// Root is the path to a directory where containerd will store persistent data
 	Root string `toml:"root"`
+	// State is the path to a directory where containerd will store transient data
+	State string `toml:"state"`
 	// GRPC configuration settings
 	GRPC GRPCConfig `toml:"grpc"`
 	// Debug and profiling settings

server/server.go

@@ -36,9 +36,15 @@ func New(ctx context.Context, config *Config) (*Server, error) {
 	if config.Root == "" {
 		return nil, errors.New("root must be specified")
 	}
+	if config.State == "" {
+		return nil, errors.New("state must be specified")
+	}
 	if err := os.MkdirAll(config.Root, 0711); err != nil {
 		return nil, err
 	}
+	if err := os.MkdirAll(config.State, 0711); err != nil {
+		return nil, err
+	}
 	if err := apply(ctx, config); err != nil {
 		return nil, err
 	}
@@ -66,6 +72,7 @@ func New(ctx context.Context, config *Config) (*Server, error) {
 			ctx,
 			initialized,
 			config.Root,
+			config.State,
 			id,
 		)
 		initContext.Events = s.events

server/server_linux.go

@@ -9,6 +9,12 @@ import (
 )
 
 const (
+	// DefaultRootDir is the default location used by containerd to store
+	// persistent data
+	DefaultRootDir = "/var/lib/containerd"
+	// DefaultStateDir is the default location used by containerd to store
+	// transient data
+	DefaultStateDir = "/run/containerd"
 	// DefaultAddress is the default unix socket address
 	DefaultAddress = "/run/containerd/containerd.sock"
 	// DefaultDebuggAddress is the default unix socket address for pprof data

server/server_unsupported.go

@@ -5,6 +5,12 @@ package server
 import "context"
 
 const (
+	// DefaultRootDir is the default location used by containerd to store
+	// persistent data
+	DefaultRootDir = "/var/lib/containerd"
+	// DefaultStateDir is the default location used by containerd to store
+	// transient data
+	DefaultStateDir = "/run/containerd"
 	// DefaultAddress is the default unix socket address
 	DefaultAddress = "/run/containerd/containerd.sock"
 	// DefaultDebuggAddress is the default unix socket address for pprof data

server/server_windows.go

@@ -2,7 +2,20 @@
 
 package server
 
-import "context"
+import (
+	"context"
+	"os"
+	"path/filepath"
+)
+
+var (
+	// DefaultRootDir is the default location used by containerd to store
+	// persistent data
+	DefaultRootDir = filepath.Join(os.Getenv("programfiles"), "containerd", "root")
+	// DefaultStateDir is the default location used by containerd to store
+	// transient data
+	DefaultStateDir = filepath.Join(os.Getenv("programfiles"), "containerd", "state")
+)
 
 const (
 	// DefaultAddress is the default winpipe address