diff --git a/cmd/ctr/run_windows.go b/cmd/ctr/run_windows.go index ded8d1f90..70f0be1cf 100644 --- a/cmd/ctr/run_windows.go +++ b/cmd/ctr/run_windows.go @@ -42,6 +42,19 @@ func spec(id string, config *ocispec.ImageConfig, context *cli.Context) *specs.S cwd = `C:\` } + // Some sane defaults for console + w := 80 + h := 20 + + if tty { + con := console.Current() + size, err := con.Size() + if err == nil { + w = int(size.Width) + h = int(size.Height) + } + } + return &specs.Spec{ Version: specs.Version, Platform: specs.Platform{ @@ -60,8 +73,8 @@ func spec(id string, config *ocispec.ImageConfig, context *cli.Context) *specs.S Username: config.User, }, ConsoleSize: specs.Box{ - Height: 20, - Width: 80, + Height: uint(w), + Width: uint(h), }, }, Hostname: id, diff --git a/windows/container.go b/windows/container.go index 8ba03b305..63a7ba382 100644 --- a/windows/container.go +++ b/windows/container.go @@ -7,146 +7,149 @@ import ( "sync" "github.com/containerd/containerd" + "github.com/containerd/containerd/log" "github.com/containerd/containerd/windows/hcs" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "golang.org/x/net/context" + winsys "golang.org/x/sys/windows" ) var ( ErrLoadedContainer = errors.New("loaded container can only be terminated") ) -type State struct { - pid uint32 - status containerd.Status -} - -func (s State) Pid() uint32 { - return s.pid -} - -func (s State) Status() containerd.Status { - return s.status -} - type eventCallback func(id string, evType containerd.EventType, pid, exitStatus uint32) -func loadContainers(ctx context.Context, rootDir string) ([]*container, error) { - hcs, err := hcs.LoadAll(ctx, owner, rootDir) +func loadContainers(ctx context.Context, h *hcs.HCS, sendEvent eventCallback) ([]*container, error) { + hCtr, err := h.LoadContainers(ctx) if err != nil { return nil, err } containers := make([]*container, 0) - for id, h := range hcs { + for _, c := range hCtr { containers = append(containers, &container{ - id: id, - status: containerd.RunningStatus, - hcs: h, + ctr: c, + status: containerd.RunningStatus, + sendEvent: sendEvent, }) } return containers, nil } -func newContainer(id, rootDir string, pid uint32, spec RuntimeSpec, io containerd.IO, sendEvent eventCallback) (*container, error) { - hcs, err := hcs.New(rootDir, owner, id, spec.OCISpec, spec.Configuration, io) +func newContainer(ctx context.Context, h *hcs.HCS, id string, spec RuntimeSpec, io containerd.IO, sendEvent eventCallback) (*container, error) { + cio, err := hcs.NewIO(io.Stdin, io.Stdout, io.Stderr, io.Terminal) if err != nil { return nil, err } + hcsCtr, err := h.CreateContainer(ctx, id, spec.OCISpec, spec.Configuration, cio) + if err != nil { + return nil, err + } + sendEvent(id, containerd.CreateEvent, hcsCtr.Pid(), 0) + return &container{ - runtimePid: pid, - id: id, - hcs: hcs, - status: containerd.CreatedStatus, - ecSync: make(chan struct{}), - sendEvent: sendEvent, + ctr: hcsCtr, + status: containerd.CreatedStatus, + sendEvent: sendEvent, }, nil } +// container implements both containerd.Container and containerd.State type container struct { sync.Mutex - runtimePid uint32 - id string - hcs *hcs.HCS - status containerd.Status - - ec uint32 - ecErr error - ecSync chan struct{} - sendEvent func(id string, evType containerd.EventType, pid, exitStatus uint32) + ctr *hcs.Container + status containerd.Status + sendEvent eventCallback } func (c *container) Info() containerd.ContainerInfo { return containerd.ContainerInfo{ - ID: c.id, + ID: c.ctr.ID(), Runtime: runtimeName, } } func (c *container) Start(ctx context.Context) error { - if c.runtimePid == 0 { + if c.ctr.Pid() == 0 { return ErrLoadedContainer } - err := c.hcs.Start(ctx, false) + err := c.ctr.Start(ctx) if err != nil { - c.hcs.Terminate(ctx) - c.sendEvent(c.id, containerd.ExitEvent, c.runtimePid, 255) return err } c.setStatus(containerd.RunningStatus) - c.sendEvent(c.id, containerd.StartEvent, c.runtimePid, 0) + c.sendEvent(c.ctr.ID(), containerd.StartEvent, c.ctr.Pid(), 0) // Wait for our process to terminate go func() { - c.ec, c.ecErr = c.hcs.ExitCode(context.Background()) + ec, err := c.ctr.ExitCode() + if err != nil { + log.G(ctx).Debug(err) + } c.setStatus(containerd.StoppedStatus) - c.sendEvent(c.id, containerd.ExitEvent, c.runtimePid, c.ec) - close(c.ecSync) + c.sendEvent(c.ctr.ID(), containerd.ExitEvent, c.ctr.Pid(), ec) }() return nil } func (c *container) State(ctx context.Context) (containerd.State, error) { - return &State{ - pid: c.runtimePid, - status: c.getStatus(), - }, nil + return c, nil } func (c *container) Kill(ctx context.Context, signal uint32, all bool) error { - return c.hcs.Terminate(ctx) + if winsys.Signal(signal) == winsys.SIGKILL { + return c.ctr.Kill(ctx) + } + return c.ctr.Stop(ctx) } func (c *container) Exec(ctx context.Context, opts containerd.ExecOpts) (containerd.Process, error) { - if c.runtimePid == 0 { + if c.ctr.Pid() == 0 { return nil, ErrLoadedContainer } + pio, err := hcs.NewIO(opts.IO.Stdin, opts.IO.Stdout, opts.IO.Stderr, opts.IO.Terminal) + if err != nil { + return nil, err + } + var procSpec specs.Process if err := json.Unmarshal(opts.Spec, &procSpec); err != nil { return nil, errors.Wrap(err, "failed to unmarshal oci spec") } - p, err := c.hcs.Exec(ctx, procSpec, opts.IO) + p, err := c.ctr.AddProcess(ctx, procSpec, pio) if err != nil { return nil, err } go func() { - ec, _ := p.ExitCode() - c.sendEvent(c.id, containerd.ExitEvent, p.Pid(), ec) + ec, err := p.ExitCode() + if err != nil { + log.G(ctx).Debug(err) + } + c.sendEvent(c.ctr.ID(), containerd.ExitEvent, p.Pid(), ec) }() return &process{p}, nil } +func (c *container) Status() containerd.Status { + return c.getStatus() +} + +func (c *container) Pid() uint32 { + return c.ctr.Pid() +} + func (c *container) setStatus(status containerd.Status) { c.Lock() c.status = status @@ -158,44 +161,3 @@ func (c *container) getStatus() containerd.Status { defer c.Unlock() return c.status } - -func (c *container) exitCode(ctx context.Context) (uint32, error) { - if c.runtimePid == 0 { - return 255, ErrLoadedContainer - } - - <-c.ecSync - return c.ec, c.ecErr -} - -func (c *container) remove(ctx context.Context) error { - return c.hcs.Remove(ctx) -} - -func (c *container) getRuntimePid() uint32 { - return c.runtimePid -} - -type process struct { - p *hcs.Process -} - -func (p *process) State(ctx context.Context) (containerd.State, error) { - return &processState{p.p}, nil -} - -func (p *process) Kill(ctx context.Context, sig uint32, all bool) error { - return p.p.Kill() -} - -type processState struct { - p *hcs.Process -} - -func (s *processState) Status() containerd.Status { - return s.p.Status() -} - -func (s *processState) Pid() uint32 { - return s.p.Pid() -} diff --git a/windows/hcs/hcs.go b/windows/hcs/hcs.go index 5cf24df3e..c682b6ae9 100644 --- a/windows/hcs/hcs.go +++ b/windows/hcs/hcs.go @@ -4,21 +4,19 @@ package hcs import ( "context" - "encoding/json" "fmt" "io" "io/ioutil" "os" "path/filepath" - "runtime" "strings" - "syscall" + "sync" "time" "github.com/Microsoft/hcsshim" "github.com/Sirupsen/logrus" - "github.com/containerd/containerd" "github.com/containerd/containerd/log" + "github.com/containerd/containerd/windows/pid" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -28,13 +26,13 @@ const ( defaultTerminateTimeout = 5 * time.Minute ) -func LoadAll(ctx context.Context, owner, rootDir string) (map[string]*HCS, error) { +func (s *HCS) LoadContainers(ctx context.Context) ([]*Container, error) { ctrProps, err := hcsshim.GetContainers(hcsshim.ComputeSystemQuery{}) if err != nil { return nil, errors.Wrap(err, "failed to retrieve running containers") } - containers := make(map[string]*HCS) + containers := make([]*Container, 0) for _, p := range ctrProps { select { case <-ctx.Done(): @@ -42,336 +40,357 @@ func LoadAll(ctx context.Context, owner, rootDir string) (map[string]*HCS, error default: } - if p.Owner != owner || p.SystemType != "Container" { + if p.Owner != s.owner || p.SystemType != "Container" { continue } - // TODO: take context in account container, err := hcsshim.OpenContainer(p.ID) if err != nil { return nil, errors.Wrapf(err, "failed open container %s", p.ID) } - stateDir := filepath.Join(rootDir, p.ID) + stateDir := filepath.Join(s.stateDir, p.ID) b, err := ioutil.ReadFile(filepath.Join(stateDir, layerFile)) - containers[p.ID] = &HCS{ + containers = append(containers, &Container{ id: p.ID, - container: container, + Container: container, stateDir: stateDir, + hcs: s, + io: &IO{}, layerFolderPath: string(b), conf: Configuration{ TerminateDuration: defaultTerminateTimeout, }, - } + }) } return containers, nil } -// New creates a new container (but doesn't start) it. -func New(rootDir, owner, containerID string, spec specs.Spec, conf Configuration, cio containerd.IO) (*HCS, error) { - stateDir := filepath.Join(rootDir, containerID) +func New(owner, rootDir string) *HCS { + return &HCS{ + stateDir: rootDir, + owner: owner, + pidPool: pid.NewPool(), + } +} + +type HCS struct { + stateDir string + owner string + pidPool *pid.Pool +} + +func (s *HCS) CreateContainer(ctx context.Context, id string, spec specs.Spec, conf Configuration, io *IO) (c *Container, err error) { + pid, err := s.pidPool.Get() + if err != nil { + return nil, err + } + defer func() { + if err != nil { + s.pidPool.Put(pid) + } + }() + + stateDir := filepath.Join(s.stateDir, id) if err := os.MkdirAll(stateDir, 0755); err != nil { return nil, errors.Wrapf(err, "unable to create container state dir %s", stateDir) } + defer func() { + if err != nil { + os.RemoveAll(stateDir) + } + }() if conf.TerminateDuration == 0 { conf.TerminateDuration = defaultTerminateTimeout } - h := &HCS{ - stateDir: stateDir, - owner: owner, - id: containerID, - spec: spec, - conf: conf, - } - - sio, err := newSIO(cio) - if err != nil { - return nil, err - } - h.io = sio - runtime.SetFinalizer(sio, func(s *shimIO) { - s.Close() - }) - - hcsConf, err := h.newHCSConfiguration() + ctrConf, err := newContainerConfig(s.owner, id, spec, conf) if err != nil { return nil, err } - ctr, err := hcsshim.CreateContainer(containerID, hcsConf) + layerPathFile := filepath.Join(stateDir, layerFile) + if err := ioutil.WriteFile(layerPathFile, []byte(ctrConf.LayerFolderPath), 0644); err != nil { + log.G(ctx).WithError(err).Warnf("failed to save active layer %s", ctrConf.LayerFolderPath) + } + + ctr, err := hcsshim.CreateContainer(id, ctrConf) if err != nil { - removeLayer(context.TODO(), hcsConf.LayerFolderPath) - return nil, err - } - h.container = ctr - h.layerFolderPath = hcsConf.LayerFolderPath - - return h, nil -} - -type HCS struct { - stateDir string - owner string - id string - spec specs.Spec - conf Configuration - io *shimIO - container hcsshim.Container - initProcess hcsshim.Process - layerFolderPath string -} - -// Start starts the associated container and instantiate the init -// process within it. -func (s *HCS) Start(ctx context.Context, servicing bool) error { - if s.initProcess != nil { - return errors.New("init process already started") - } - if err := s.container.Start(); err != nil { - if err := s.Terminate(ctx); err != nil { - log.G(ctx).WithError(err).Errorf("failed to terminate container %s", s.id) - } - return err + removeLayer(ctx, ctrConf.LayerFolderPath) + return nil, errors.Wrapf(err, "failed to create container %s", id) } - proc, err := s.newProcess(ctx, s.io, s.spec.Process) + err = ctr.Start() if err != nil { - s.Terminate(ctx) - return err + ctr.Terminate() + removeLayer(ctx, ctrConf.LayerFolderPath) + return nil, errors.Wrapf(err, "failed to start container %s", id) } - s.initProcess = proc - - return nil -} - -// Pid returns the pid of the container init process -func (s *HCS) Pid() int { - return s.initProcess.Pid() -} - -// ExitCode waits for the container to exit and return the exit code -// of the init process -func (s *HCS) ExitCode(ctx context.Context) (uint32, error) { - // TODO: handle a context cancellation - if err := s.initProcess.Wait(); err != nil { - if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE { - return 255, errors.Wrapf(err, "failed to wait for container '%s' init process", s.id) - } - // container is probably dead, let's try to get its exit code - } - - ec, err := s.initProcess.ExitCode() - if err != nil { - if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE { - return 255, errors.Wrapf(err, "failed to get container '%s' init process exit code", s.id) - } - // Well, unknown exit code it is - ec = 255 - } - - return uint32(ec), err -} - -// Exec starts a new process within the container -func (s *HCS) Exec(ctx context.Context, procSpec specs.Process, io containerd.IO) (*Process, error) { - sio, err := newSIO(io) - if err != nil { - return nil, err - } - p, err := s.newProcess(ctx, sio, procSpec) - if err != nil { - return nil, err - } - - return &Process{ - containerID: s.id, - p: p, - status: containerd.RunningStatus, + return &Container{ + Container: ctr, + id: id, + pid: pid, + spec: spec, + conf: conf, + stateDir: stateDir, + io: io, + hcs: s, + layerFolderPath: ctrConf.LayerFolderPath, + processes: make([]*Process, 0), }, nil } -// newProcess create a new process within a running container. This is -// used to create both the init process and subsequent 'exec' -// processes. -func (s *HCS) newProcess(ctx context.Context, sio *shimIO, procSpec specs.Process) (hcsshim.Process, error) { - conf := hcsshim.ProcessConfig{ - EmulateConsole: sio.terminal, - CreateStdInPipe: sio.stdin != nil, - CreateStdOutPipe: sio.stdout != nil, - CreateStdErrPipe: sio.stderr != nil, - User: procSpec.User.Username, - CommandLine: strings.Join(procSpec.Args, " "), - Environment: ociSpecEnvToHCSEnv(procSpec.Env), - WorkingDirectory: procSpec.Cwd, +type Container struct { + sync.Mutex + hcsshim.Container + + id string + stateDir string + pid uint32 + spec specs.Spec + conf Configuration + io *IO + hcs *HCS + layerFolderPath string + + processes []*Process +} + +func (c *Container) ID() string { + return c.id +} + +func (c *Container) Pid() uint32 { + return c.pid +} + +func (c *Container) Start(ctx context.Context) error { + _, err := c.addProcess(ctx, c.spec.Process, c.io) + return err +} + +func (c *Container) getDeathErr(err error) error { + switch { + case hcsshim.IsPending(err): + err = c.WaitTimeout(c.conf.TerminateDuration) + case hcsshim.IsAlreadyStopped(err): + err = nil + } + return err +} + +func (c *Container) Kill(ctx context.Context) error { + return c.getDeathErr(c.Terminate()) +} + +func (c *Container) Stop(ctx context.Context) error { + err := c.getDeathErr(c.Shutdown()) + if err != nil { + log.G(ctx).WithError(err).Debugf("failed to shutdown container %s, calling terminate", c.id) + return c.getDeathErr(c.Terminate()) + } + return nil +} + +func (c *Container) Delete(ctx context.Context) { + defer func() { + if err := c.Stop(ctx); err != nil { + log.G(ctx).WithError(err).WithField("id", c.id). + Errorf("failed to shutdown/terminate container") + } + + c.Lock() + for _, p := range c.processes { + if err := p.Delete(); err != nil { + log.G(ctx).WithError(err).WithFields(logrus.Fields{"pid": p.Pid(), "id": c.id}). + Errorf("failed to clean process resources") + } + } + c.Unlock() + + if err := c.Close(); err != nil { + log.G(ctx).WithError(err).WithField("id", c.id).Errorf("failed to clean container resources") + } + + c.io.Close() + + // Cleanup folder layer + if err := removeLayer(ctx, c.layerFolderPath); err == nil { + os.RemoveAll(c.stateDir) + } + }() + + if update, err := c.HasPendingUpdates(); err != nil || !update { + return + } + + serviceCtr, err := c.hcs.CreateContainer(ctx, c.id+"_servicing", c.spec, c.conf, &IO{}) + if err != nil { + log.G(ctx).WithError(err).WithField("id", c.id).Warn("could not create servicing container") + return + } + defer serviceCtr.Close() + + err = serviceCtr.Start(ctx) + if err != nil { + log.G(ctx).WithError(err).WithField("id", c.id).Warn("failed to start servicing container") + serviceCtr.Terminate() + return + } + + err = serviceCtr.processes[0].Wait() + if err == nil { + _, err = serviceCtr.processes[0].ExitCode() + log.G(ctx).WithError(err).WithField("id", c.id).Errorf("failed to retrieve servicing container exit code") + } + + if err != nil { + if err := serviceCtr.Terminate(); err != nil { + log.G(ctx).WithError(err).WithField("id", c.id).Errorf("failed to terminate servicing container") + } + } +} + +func (c *Container) ExitCode() (uint32, error) { + if len(c.processes) == 0 { + return 255, errors.New("container not started") + } + return c.processes[0].ExitCode() +} + +func (c *Container) AddProcess(ctx context.Context, spec specs.Process, io *IO) (*Process, error) { + if len(c.processes) == 0 { + return nil, errors.New("container not started") + } + + return c.addProcess(ctx, spec, io) +} + +func (c *Container) addProcess(ctx context.Context, spec specs.Process, pio *IO) (*Process, error) { + // If we don't have a process yet, reused the container pid + var pid uint32 + if len(c.processes) == 0 { + pid = c.pid + } else { + pid, err := c.hcs.pidPool.Get() + if err != nil { + return nil, err + } + defer func() { + if err != nil { + c.hcs.pidPool.Put(pid) + } + }() + } + + conf := hcsshim.ProcessConfig{ + EmulateConsole: pio.terminal, + CreateStdInPipe: pio.stdin != nil, + CreateStdOutPipe: pio.stdout != nil, + CreateStdErrPipe: pio.stderr != nil, + User: spec.User.Username, + CommandLine: strings.Join(spec.Args, " "), + Environment: ociSpecEnvToHCSEnv(spec.Env), + WorkingDirectory: spec.Cwd, + ConsoleSize: [2]uint{spec.ConsoleSize.Height, spec.ConsoleSize.Width}, } - conf.ConsoleSize[0] = procSpec.ConsoleSize.Height - conf.ConsoleSize[1] = procSpec.ConsoleSize.Width if conf.WorkingDirectory == "" { - conf.WorkingDirectory = s.spec.Process.Cwd + conf.WorkingDirectory = c.spec.Process.Cwd } - proc, err := s.container.CreateProcess(&conf) + proc, err := c.CreateProcess(&conf) if err != nil { - return nil, errors.Wrapf(err, "failed to create process with conf %#v", conf) - + return nil, errors.Wrapf(err, "failed to create process") } - pid := proc.Pid() stdin, stdout, stderr, err := proc.Stdio() if err != nil { - s.Terminate(ctx) - return nil, err + proc.Kill() + return nil, errors.Wrapf(err, "failed to retrieve process stdio") } - if sio.stdin != nil { + if pio.stdin != nil { go func() { - log.G(ctx).WithField("pid", pid).Debug("stdin: copy started") - io.Copy(stdin, sio.stdin) - log.G(ctx).WithField("pid", pid).Debug("stdin: copy done") + log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdin: copy started") + io.Copy(stdin, pio.stdin) + log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdin: copy done") stdin.Close() - sio.stdin.Close() + pio.stdin.Close() }() } else { proc.CloseStdin() } - if sio.stdout != nil { + if pio.stdout != nil { go func() { - log.G(ctx).WithField("pid", pid).Debug("stdout: copy started") - io.Copy(sio.stdout, stdout) - log.G(ctx).WithField("pid", pid).Debug("stdout: copy done") + log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdout: copy started") + io.Copy(pio.stdout, stdout) + log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdout: copy done") stdout.Close() - sio.stdout.Close() + pio.stdout.Close() }() } - if sio.stderr != nil { + if pio.stderr != nil { go func() { - log.G(ctx).WithField("pid", pid).Debug("stderr: copy started") - io.Copy(sio.stderr, stderr) - log.G(ctx).WithField("pid", pid).Debug("stderr: copy done") + log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stderr: copy started") + io.Copy(pio.stderr, stderr) + log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stderr: copy done") stderr.Close() - sio.stderr.Close() + pio.stderr.Close() }() } - return proc, nil -} - -// Terminate stop a running container. -func (s *HCS) Terminate(ctx context.Context) error { - err := s.container.Terminate() - switch { - case hcsshim.IsPending(err): - // TODO: take the context into account - err = s.container.WaitTimeout(s.conf.TerminateDuration) - case hcsshim.IsAlreadyStopped(err): - err = nil + p := &Process{ + Process: proc, + pid: pid, + io: pio, + ecSync: make(chan struct{}), } - return err -} + c.Lock() + c.processes = append(c.processes, p) + idx := len(c.processes) - 1 + c.Unlock() -func (s *HCS) Shutdown(ctx context.Context) error { - err := s.container.Shutdown() - switch { - case hcsshim.IsPending(err): - // TODO: take the context into account - err = s.container.WaitTimeout(s.conf.TerminateDuration) - case hcsshim.IsAlreadyStopped(err): - err = nil - } - - if err != nil { - log.G(ctx).WithError(err).Debugf("failed to shutdown container %s, calling terminate", s.id) - return s.Terminate(ctx) - } - - return nil -} - -// Remove start a servicing container if needed then cleanup the container -// resources -func (s *HCS) Remove(ctx context.Context) error { - defer func() { - if err := s.Shutdown(ctx); err != nil { - log.G(ctx).WithError(err).WithField("id", s.id). - Errorf("failed to shutdown/terminate container") - } - - if s.initProcess != nil { - if err := s.initProcess.Close(); err != nil { - log.G(ctx).WithError(err).WithFields(logrus.Fields{"pid": s.Pid(), "id": s.id}). - Errorf("failed to clean init process resources") - } - } - if err := s.container.Close(); err != nil { - log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to clean container resources") - } - - // Cleanup folder layer - if err := removeLayer(ctx, s.layerFolderPath); err == nil { - os.RemoveAll(s.stateDir) + go func() { + p.ec, p.ecErr = processExitCode(c.ID(), p) + close(p.ecSync) + c.Lock() + p.Delete() + // Remove process from slice (but keep the init one around) + if idx > 0 { + c.processes[idx] = c.processes[len(c.processes)-1] + c.processes[len(c.processes)-1] = nil + c.processes = c.processes[:len(c.processes)-1] } + c.Unlock() }() - if update, err := s.container.HasPendingUpdates(); err != nil || !update { - return nil - } - - // TODO: take the context into account - serviceHCS, err := New(s.stateDir, s.owner, s.id+"_servicing", s.spec, s.conf, containerd.IO{}) - if err != nil { - log.G(ctx).WithError(err).WithField("id", s.id).Warn("could not create servicing container") - return nil - } - defer serviceHCS.container.Close() - - err = serviceHCS.Start(ctx, true) - if err != nil { - if err := serviceHCS.Terminate(ctx); err != nil { - log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to terminate servicing container for %s") - } - log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to start servicing container") - return nil - } - - // wait for the container to exit - _, err = serviceHCS.ExitCode(ctx) - if err != nil { - if err := serviceHCS.Terminate(ctx); err != nil { - log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to terminate servicing container for %s") - } - log.G(ctx).WithError(err).WithField("id", s.id).Errorf("failed to get servicing container exit code") - } - - serviceHCS.container.WaitTimeout(s.conf.TerminateDuration) - - return nil + return p, nil } // newHCSConfiguration generates a hcsshim configuration from the instance // OCI Spec and hcs.Configuration. -func (s *HCS) newHCSConfiguration() (*hcsshim.ContainerConfig, error) { +func newContainerConfig(owner, id string, spec specs.Spec, conf Configuration) (*hcsshim.ContainerConfig, error) { configuration := &hcsshim.ContainerConfig{ SystemType: "Container", - Name: s.id, - Owner: s.owner, - HostName: s.spec.Hostname, - IgnoreFlushesDuringBoot: s.conf.IgnoreFlushesDuringBoot, - HvPartition: s.conf.UseHyperV, - AllowUnqualifiedDNSQuery: s.conf.AllowUnqualifiedDNSQuery, - EndpointList: s.conf.NetworkEndpoints, - NetworkSharedContainerName: s.conf.NetworkSharedContainerID, - Credentials: s.conf.Credentials, + Name: id, + Owner: owner, + HostName: spec.Hostname, + IgnoreFlushesDuringBoot: conf.IgnoreFlushesDuringBoot, + HvPartition: conf.UseHyperV, + AllowUnqualifiedDNSQuery: conf.AllowUnqualifiedDNSQuery, + EndpointList: conf.NetworkEndpoints, + NetworkSharedContainerName: conf.NetworkSharedContainerID, + Credentials: conf.Credentials, } // TODO: use the create request Mount for those - for _, layerPath := range s.conf.Layers { + for _, layerPath := range conf.Layers { _, filename := filepath.Split(layerPath) guid, err := hcsshim.NameToGuid(filename) if err != nil { @@ -383,9 +402,9 @@ func (s *HCS) newHCSConfiguration() (*hcsshim.ContainerConfig, error) { }) } - if len(s.spec.Mounts) > 0 { - mds := make([]hcsshim.MappedDir, len(s.spec.Mounts)) - for i, mount := range s.spec.Mounts { + if len(spec.Mounts) > 0 { + mds := make([]hcsshim.MappedDir, len(spec.Mounts)) + for i, mount := range spec.Mounts { mds[i] = hcsshim.MappedDir{ HostPath: mount.Source, ContainerPath: mount.Destination, @@ -400,12 +419,12 @@ func (s *HCS) newHCSConfiguration() (*hcsshim.ContainerConfig, error) { configuration.MappedDirectories = mds } - if s.conf.DNSSearchList != nil { - configuration.DNSSearchList = strings.Join(s.conf.DNSSearchList, ",") + if conf.DNSSearchList != nil { + configuration.DNSSearchList = strings.Join(conf.DNSSearchList, ",") } if configuration.HvPartition { - for _, layerPath := range s.conf.Layers { + for _, layerPath := range conf.Layers { utilityVMPath := filepath.Join(layerPath, "UtilityVM") _, err := os.Stat(utilityVMPath) if err == nil { @@ -427,55 +446,42 @@ func (s *HCS) newHCSConfiguration() (*hcsshim.ContainerConfig, error) { } if len(configuration.Layers) > 0 { - di.HomeDir = filepath.Dir(s.conf.Layers[0]) + di.HomeDir = filepath.Dir(conf.Layers[0]) } // Windows doesn't support creating a container with a readonly // filesystem, so always create a RW one - if err := hcsshim.CreateSandboxLayer(di, s.id, s.conf.Layers[0], s.conf.Layers); err != nil { + if err := hcsshim.CreateSandboxLayer(di, id, conf.Layers[0], conf.Layers); err != nil { return nil, errors.Wrapf(err, "failed to create sandbox layer for %s: layers: %#v, driverInfo: %#v", - s.id, configuration.Layers, di) + id, configuration.Layers, di) } + configuration.LayerFolderPath = filepath.Join(di.HomeDir, id) - configuration.LayerFolderPath = filepath.Join(di.HomeDir, s.id) - if err := ioutil.WriteFile(filepath.Join(s.stateDir, layerFile), []byte(configuration.LayerFolderPath), 0644); err != nil { - log.L.WithError(err).Warnf("failed to save active layer %s", configuration.LayerFolderPath) - } - - err := hcsshim.ActivateLayer(di, s.id) + err := hcsshim.ActivateLayer(di, id) if err != nil { removeLayer(context.TODO(), configuration.LayerFolderPath) return nil, errors.Wrapf(err, "failed to active layer %s", configuration.LayerFolderPath) } - err = hcsshim.PrepareLayer(di, s.id, s.conf.Layers) + err = hcsshim.PrepareLayer(di, id, conf.Layers) if err != nil { removeLayer(context.TODO(), configuration.LayerFolderPath) return nil, errors.Wrapf(err, "failed to prepare layer %s", configuration.LayerFolderPath) } - volumePath, err := hcsshim.GetLayerMountPath(di, s.id) + volumePath, err := hcsshim.GetLayerMountPath(di, id) if err != nil { - if err := hcsshim.DestroyLayer(di, s.id); err != nil { - log.L.Warnf("failed to DestroyLayer %s: %s", s.id, err) + if err := hcsshim.DestroyLayer(di, id); err != nil { + log.L.Warnf("failed to DestroyLayer %s: %s", id, err) } - return nil, errors.Wrapf(err, "failed to getmount path for layer %s: driverInfo: %#v", s.id, di) + return nil, errors.Wrapf(err, "failed to getmount path for layer %s: driverInfo: %#v", id, di) } configuration.VolumePath = volumePath - f, err := os.OpenFile(fmt.Sprintf("%s-hcs.json", s.id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 066) - if err != nil { - fmt.Println("failed to create file:", err) - } else { - defer f.Close() - enc := json.NewEncoder(f) - enc.Encode(configuration) - } - return configuration, nil } -// removeLayer delete the given layer, all associated containers must have +// removeLayer deletes the given layer, all associated containers must have // been shutdown for this to succeed. func removeLayer(ctx context.Context, path string) error { layerID := filepath.Base(path) diff --git a/windows/hcs/process.go b/windows/hcs/process.go index 940033e68..4229e6416 100644 --- a/windows/hcs/process.go +++ b/windows/hcs/process.go @@ -11,32 +11,51 @@ import ( ) type Process struct { - containerID string - p hcsshim.Process - status containerd.Status + hcsshim.Process + + pid uint32 + io *IO + ec uint32 + ecErr error + ecSync chan struct{} } -func (h *Process) Pid() uint32 { - return uint32(h.p.Pid()) +func (p *Process) Pid() uint32 { + return p.pid } -func (h *Process) Kill() error { - return h.p.Kill() +func (p *Process) ExitCode() (uint32, error) { + <-p.ecSync + return p.ec, p.ecErr } -func (h *Process) ExitCode() (uint32, error) { - if err := h.p.Wait(); err != nil { - if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE { - return 255, errors.Wrapf(err, "failed to wait for container '%s' process %d", h.containerID, h.p.Pid()) - } - // container is probably dead, let's try to get its exit code +func (p *Process) Status() containerd.Status { + select { + case <-p.ecSync: + return containerd.StoppedStatus + default: } - h.status = containerd.StoppedStatus - ec, err := h.p.ExitCode() + return containerd.RunningStatus +} + +func (p *Process) Delete() error { + p.io.Close() + return p.Close() +} + +func processExitCode(containerID string, p *Process) (uint32, error) { + if err := p.Wait(); err != nil { + if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE { + return 255, errors.Wrapf(err, "failed to wait for container '%s' process %u", containerID, p.pid) + } + // process is probably dead, let's try to get its exit code + } + + ec, err := p.Process.ExitCode() if err != nil { if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE { - return 255, errors.Wrapf(err, "failed to get container '%s' process %d exit code", h.containerID, h.p.Pid()) + return 255, errors.Wrapf(err, "failed to get container '%s' process %d exit code", containerID, p.pid) } // Well, unknown exit code it is ec = 255 @@ -44,7 +63,3 @@ func (h *Process) ExitCode() (uint32, error) { return uint32(ec), err } - -func (h *Process) Status() containerd.Status { - return h.status -} diff --git a/windows/hcs/shimio.go b/windows/hcs/shimio.go index 2dfb5e28c..76e1d57fd 100644 --- a/windows/hcs/shimio.go +++ b/windows/hcs/shimio.go @@ -7,28 +7,27 @@ import ( "time" "github.com/Microsoft/go-winio" - "github.com/containerd/containerd" "github.com/pkg/errors" ) -type shimIO struct { +type IO struct { stdin net.Conn stdout net.Conn stderr net.Conn terminal bool } -// newSIO connects to the provided pipes -func newSIO(io containerd.IO) (*shimIO, error) { +// NewIO connects to the provided pipe addresses +func NewIO(stdin, stdout, stderr string, terminal bool) (*IO, error) { var ( c net.Conn err error - sio shimIO + io IO ) defer func() { if err != nil { - sio.Close() + io.Close() } }() @@ -38,19 +37,19 @@ func newSIO(io containerd.IO) (*shimIO, error) { conn *net.Conn }{ { - name: io.Stdin, - open: io.Stdin != "", - conn: &sio.stdin, + name: stdin, + open: stdin != "", + conn: &io.stdin, }, { - name: io.Stdout, - open: io.Stdout != "", - conn: &sio.stdout, + name: stdout, + open: stdout != "", + conn: &io.stdout, }, { - name: io.Stderr, - open: !io.Terminal && io.Stderr != "", - conn: &sio.stderr, + name: stderr, + open: !terminal && stderr != "", + conn: &io.stderr, }, } { if p.open { @@ -63,12 +62,12 @@ func newSIO(io containerd.IO) (*shimIO, error) { } } - return &sio, nil + return &io, nil } // Close terminates all successfully dialed IO connections -func (s *shimIO) Close() { - for _, cn := range []net.Conn{s.stdin, s.stdout, s.stderr} { +func (i *IO) Close() { + for _, cn := range []net.Conn{i.stdin, i.stdout, i.stderr} { if cn != nil { cn.Close() cn = nil diff --git a/windows/pid/pid.go b/windows/pid/pid.go new file mode 100644 index 000000000..503b3b941 --- /dev/null +++ b/windows/pid/pid.go @@ -0,0 +1,46 @@ +// +build windows + +package pid + +import ( + "errors" + "sync" +) + +type Pool struct { + sync.Mutex + pool map[uint32]struct{} + cur uint32 +} + +func NewPool() *Pool { + return &Pool{ + pool: make(map[uint32]struct{}), + } +} + +func (p *Pool) Get() (uint32, error) { + p.Lock() + defer p.Unlock() + + pid := p.cur + 1 + for pid != p.cur { + // 0 is reserved and invalid + if pid == 0 { + pid = 1 + } + if _, ok := p.pool[pid]; !ok { + p.cur = pid + return pid, nil + } + pid++ + } + + return 0, errors.New("pid pool exhausted") +} + +func (p *Pool) Put(pid uint32) { + p.Lock() + delete(p.pool, pid) + p.Unlock() +} diff --git a/windows/process.go b/windows/process.go new file mode 100644 index 000000000..c6cec4ee7 --- /dev/null +++ b/windows/process.go @@ -0,0 +1,30 @@ +// +build windows + +package windows + +import ( + "github.com/containerd/containerd" + "github.com/containerd/containerd/windows/hcs" + "golang.org/x/net/context" +) + +// process implements containerd.Process and containerd.State +type process struct { + *hcs.Process +} + +func (p *process) State(ctx context.Context) (containerd.State, error) { + return p, nil +} + +func (p *process) Kill(ctx context.Context, sig uint32, all bool) error { + return p.Process.Kill() +} + +func (p *process) Status() containerd.Status { + return p.Process.Status() +} + +func (p *process) Pid() uint32 { + return p.Process.Pid() +} diff --git a/windows/runtime.go b/windows/runtime.go index 138f4396c..58751cf28 100644 --- a/windows/runtime.go +++ b/windows/runtime.go @@ -8,13 +8,13 @@ import ( "os" "path/filepath" "sync" - "syscall" "time" "github.com/containerd/containerd" "github.com/containerd/containerd/log" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/windows/hcs" + "github.com/containerd/containerd/windows/pid" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -22,20 +22,11 @@ import ( ) const ( - runtimeName = "windows" - owner = "containerd" - configFilename = "config.json" + runtimeName = "windows" + owner = "containerd" ) -// Win32 error codes that are used for various workarounds -// These really should be ALL_CAPS to match golangs syscall library and standard -// Win32 error conventions, but golint insists on CamelCase. -const ( - CoEClassstring = syscall.Errno(0x800401F3) // Invalid class string - ErrorNoNetwork = syscall.Errno(1222) // The network is not present or not started - ErrorBadPathname = syscall.Errno(161) // The specified path is invalid - ErrorInvalidObject = syscall.Errno(0x800710D8) // The object identifier does not represent a valid object -) +var _ = (containerd.Runtime)(&Runtime{}) func init() { plugin.Register(runtimeName, &plugin.Registration{ @@ -52,16 +43,30 @@ func New(ic *plugin.InitContext) (interface{}, error) { return nil, errors.Wrapf(err, "could not create state directory at %s", rootDir) } + r := &Runtime{ + pidPool: pid.NewPool(), + containers: make(map[string]*container), + events: make(chan *containerd.Event, 2048), + eventsContext: c, + eventsCancel: cancel, + rootDir: rootDir, + hcs: hcs.New(owner, rootDir), + } + + sendEvent := func(id string, evType containerd.EventType, pid, exitStatus uint32) { + r.sendEvent(id, evType, pid, exitStatus) + } + // Terminate all previous container that we may have started. We don't // support restoring containers - - ctrs, err := loadContainers(ic.Context, rootDir) + ctrs, err := loadContainers(ic.Context, r.hcs, sendEvent) if err != nil { return nil, err } for _, c := range ctrs { - c.remove(ic.Context) + c.ctr.Delete(ic.Context) + r.sendEvent(c.ctr.ID(), containerd.ExitEvent, c.ctr.Pid(), 255) } // Try to delete the old state dir and recreate it @@ -72,16 +77,9 @@ func New(ic *plugin.InitContext) (interface{}, error) { if err := os.MkdirAll(stateDir, 0755); err != nil { return nil, errors.Wrapf(err, "could not create state directory at %s", stateDir) } + r.stateDir = stateDir - return &Runtime{ - containers: make(map[string]*container), - containersPid: make(map[uint32]struct{}), - events: make(chan *containerd.Event, 2048), - eventsContext: c, - eventsCancel: cancel, - stateDir: stateDir, - rootDir: rootDir, - }, nil + return r, nil } type Runtime struct { @@ -89,10 +87,11 @@ type Runtime struct { rootDir string stateDir string + pidPool *pid.Pool - containers map[string]*container - containersPid map[uint32]struct{} - currentPid uint32 + hcs *hcs.HCS + + containers map[string]*container events chan *containerd.Event eventsContext context.Context @@ -113,26 +112,19 @@ func (r *Runtime) Create(ctx context.Context, id string, opts containerd.CreateO return nil, errors.Wrap(err, "failed to unmarshal oci spec") } - pid, err := r.getPid() - if err != nil { - return nil, err + sendEvent := func(id string, evType containerd.EventType, pid, exitStatus uint32) { + r.sendEvent(id, evType, pid, exitStatus) } - ctr, err := newContainer(id, r.rootDir, pid, rtSpec, opts.IO, func(id string, evType containerd.EventType, pid, exitStatus uint32) { - r.sendEvent(id, evType, pid, exitStatus) - }) + ctr, err := newContainer(ctx, r.hcs, id, rtSpec, opts.IO, sendEvent) if err != nil { - r.putPid(pid) return nil, err } r.Lock() r.containers[id] = ctr - r.containersPid[pid] = struct{}{} r.Unlock() - r.sendEvent(id, containerd.CreateEvent, pid, 0) - return ctr, nil } @@ -141,30 +133,32 @@ func (r *Runtime) Delete(ctx context.Context, c containerd.Container) (uint32, e if !ok { return 0, fmt.Errorf("container cannot be cast as *windows.container") } - ec, err := wc.exitCode(ctx) + ec, err := wc.ctr.ExitCode() if err != nil { - ec = 255 - log.G(ctx).WithError(err).Errorf("failed to retrieve exit code for container %s", c.Info().ID) + log.G(ctx).WithError(err).Errorf("failed to retrieve exit code for container %s", wc.ctr.ID()) } - if err = wc.remove(ctx); err == nil { - r.Lock() - delete(r.containers, c.Info().ID) - r.Unlock() - } + wc.ctr.Delete(ctx) - r.putPid(wc.getRuntimePid()) + r.Lock() + delete(r.containers, wc.ctr.ID()) + r.Unlock() return ec, err } -func (r *Runtime) Containers() ([]containerd.Container, error) { +func (r *Runtime) Containers(ctx context.Context) ([]containerd.Container, error) { r.Lock() + defer r.Unlock() list := make([]containerd.Container, len(r.containers)) for _, c := range r.containers { - list = append(list, c) + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + list = append(list, c) + } } - r.Unlock() return list, nil } @@ -183,29 +177,3 @@ func (r *Runtime) sendEvent(id string, evType containerd.EventType, pid, exitSta ExitStatus: exitStatus, } } - -func (r *Runtime) getPid() (uint32, error) { - r.Lock() - defer r.Unlock() - - pid := r.currentPid + 1 - for pid != r.currentPid { - // 0 is reserved and invalid - if pid == 0 { - pid = 1 - } - if _, ok := r.containersPid[pid]; !ok { - r.currentPid = pid - return pid, nil - } - pid++ - } - - return 0, errors.New("pid pool exhausted") -} - -func (r *Runtime) putPid(pid uint32) { - r.Lock() - delete(r.containersPid, pid) - r.Unlock() -}