573 lines
14 KiB
Go
573 lines
14 KiB
Go
// +build windows
|
|
|
|
package hcs
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/Microsoft/hcsshim"
|
|
"github.com/Sirupsen/logrus"
|
|
"github.com/containerd/containerd/log"
|
|
"github.com/containerd/containerd/plugin"
|
|
"github.com/containerd/containerd/windows/pid"
|
|
"github.com/opencontainers/runtime-spec/specs-go"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
const (
|
|
layerFile = "layer"
|
|
defaultTerminateTimeout = 5 * time.Minute
|
|
)
|
|
|
|
func (s *HCS) LoadContainers(ctx context.Context) ([]*Container, error) {
|
|
ctrProps, err := hcsshim.GetContainers(hcsshim.ComputeSystemQuery{})
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to retrieve running containers")
|
|
}
|
|
|
|
containers := make([]*Container, 0)
|
|
for _, p := range ctrProps {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
if p.Owner != s.owner || p.SystemType != "Container" {
|
|
continue
|
|
}
|
|
|
|
container, err := hcsshim.OpenContainer(p.ID)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed open container %s", p.ID)
|
|
}
|
|
stateDir := filepath.Join(s.stateDir, p.ID)
|
|
b, err := ioutil.ReadFile(filepath.Join(stateDir, layerFile))
|
|
containers = append(containers, &Container{
|
|
id: p.ID,
|
|
Container: container,
|
|
stateDir: stateDir,
|
|
hcs: s,
|
|
io: &IO{},
|
|
layerFolderPath: string(b),
|
|
conf: Configuration{
|
|
TerminateDuration: defaultTerminateTimeout,
|
|
},
|
|
})
|
|
}
|
|
|
|
return containers, nil
|
|
}
|
|
|
|
func New(owner, rootDir string) *HCS {
|
|
return &HCS{
|
|
stateDir: rootDir,
|
|
owner: owner,
|
|
pidPool: pid.NewPool(),
|
|
}
|
|
}
|
|
|
|
type HCS struct {
|
|
stateDir string
|
|
owner string
|
|
pidPool *pid.Pool
|
|
}
|
|
|
|
func (s *HCS) CreateContainer(ctx context.Context, id string, spec specs.Spec, conf Configuration, io *IO) (c *Container, err error) {
|
|
pid, err := s.pidPool.Get()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
if err != nil {
|
|
s.pidPool.Put(pid)
|
|
}
|
|
}()
|
|
|
|
stateDir := filepath.Join(s.stateDir, id)
|
|
if err := os.MkdirAll(stateDir, 0755); err != nil {
|
|
return nil, errors.Wrapf(err, "unable to create container state dir %s", stateDir)
|
|
}
|
|
defer func() {
|
|
if err != nil {
|
|
os.RemoveAll(stateDir)
|
|
}
|
|
}()
|
|
|
|
if conf.TerminateDuration == 0 {
|
|
conf.TerminateDuration = defaultTerminateTimeout
|
|
}
|
|
|
|
ctrConf, err := newContainerConfig(s.owner, id, spec, conf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
layerPathFile := filepath.Join(stateDir, layerFile)
|
|
if err := ioutil.WriteFile(layerPathFile, []byte(ctrConf.LayerFolderPath), 0644); err != nil {
|
|
log.G(ctx).WithError(err).Warnf("failed to save active layer %s", ctrConf.LayerFolderPath)
|
|
}
|
|
|
|
ctr, err := hcsshim.CreateContainer(id, ctrConf)
|
|
if err != nil {
|
|
removeLayer(ctx, ctrConf.LayerFolderPath)
|
|
return nil, errors.Wrapf(err, "failed to create container %s", id)
|
|
}
|
|
|
|
err = ctr.Start()
|
|
if err != nil {
|
|
ctr.Terminate()
|
|
removeLayer(ctx, ctrConf.LayerFolderPath)
|
|
return nil, errors.Wrapf(err, "failed to start container %s", id)
|
|
}
|
|
|
|
return &Container{
|
|
Container: ctr,
|
|
id: id,
|
|
pid: pid,
|
|
spec: spec,
|
|
conf: conf,
|
|
stateDir: stateDir,
|
|
io: io,
|
|
hcs: s,
|
|
layerFolderPath: ctrConf.LayerFolderPath,
|
|
processes: make([]*Process, 0),
|
|
}, nil
|
|
}
|
|
|
|
type Container struct {
|
|
sync.Mutex
|
|
hcsshim.Container
|
|
|
|
id string
|
|
stateDir string
|
|
pid uint32
|
|
spec specs.Spec
|
|
conf Configuration
|
|
io *IO
|
|
hcs *HCS
|
|
layerFolderPath string
|
|
|
|
processes []*Process
|
|
}
|
|
|
|
func (c *Container) ID() string {
|
|
return c.id
|
|
}
|
|
|
|
func (c *Container) Pid() uint32 {
|
|
return c.pid
|
|
}
|
|
|
|
func (c *Container) Processes() []*Process {
|
|
return c.processes
|
|
}
|
|
|
|
func (c *Container) Start(ctx context.Context) error {
|
|
_, err := c.addProcess(ctx, c.spec.Process, c.io)
|
|
return err
|
|
}
|
|
|
|
func (c *Container) getDeathErr(err error) error {
|
|
switch {
|
|
case hcsshim.IsPending(err):
|
|
err = c.WaitTimeout(c.conf.TerminateDuration)
|
|
case hcsshim.IsAlreadyStopped(err):
|
|
err = nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (c *Container) Kill(ctx context.Context) error {
|
|
return c.getDeathErr(c.Terminate())
|
|
}
|
|
|
|
func (c *Container) Stop(ctx context.Context) error {
|
|
err := c.getDeathErr(c.Shutdown())
|
|
if err != nil {
|
|
log.G(ctx).WithError(err).Debugf("failed to shutdown container %s, calling terminate", c.id)
|
|
return c.getDeathErr(c.Terminate())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Container) CloseStdin(ctx context.Context, pid uint32) error {
|
|
var proc *Process
|
|
c.Lock()
|
|
for _, p := range c.processes {
|
|
if p.Pid() == pid {
|
|
proc = p
|
|
break
|
|
}
|
|
}
|
|
c.Unlock()
|
|
if proc == nil {
|
|
return errors.Errorf("no such process %v", pid)
|
|
}
|
|
|
|
return proc.CloseStdin()
|
|
}
|
|
|
|
func (c *Container) Pty(ctx context.Context, pid uint32, size plugin.ConsoleSize) error {
|
|
var proc *Process
|
|
c.Lock()
|
|
for _, p := range c.processes {
|
|
if p.Pid() == pid {
|
|
proc = p
|
|
break
|
|
}
|
|
}
|
|
c.Unlock()
|
|
if proc == nil {
|
|
return errors.Errorf("no such process %v", pid)
|
|
}
|
|
|
|
return proc.ResizeConsole(uint16(size.Width), uint16(size.Height))
|
|
}
|
|
|
|
func (c *Container) Delete(ctx context.Context) {
|
|
defer func() {
|
|
if err := c.Stop(ctx); err != nil {
|
|
log.G(ctx).WithError(err).WithField("id", c.id).
|
|
Errorf("failed to shutdown/terminate container")
|
|
}
|
|
|
|
c.Lock()
|
|
for _, p := range c.processes {
|
|
if err := p.Delete(); err != nil {
|
|
log.G(ctx).WithError(err).WithFields(logrus.Fields{"pid": p.Pid(), "id": c.id}).
|
|
Errorf("failed to clean process resources")
|
|
}
|
|
}
|
|
c.Unlock()
|
|
|
|
if err := c.Close(); err != nil {
|
|
log.G(ctx).WithError(err).WithField("id", c.id).Errorf("failed to clean container resources")
|
|
}
|
|
|
|
c.io.Close()
|
|
|
|
// Cleanup folder layer
|
|
if err := removeLayer(ctx, c.layerFolderPath); err == nil {
|
|
os.RemoveAll(c.stateDir)
|
|
}
|
|
}()
|
|
|
|
if update, err := c.HasPendingUpdates(); err != nil || !update {
|
|
return
|
|
}
|
|
|
|
serviceCtr, err := c.hcs.CreateContainer(ctx, c.id+"_servicing", c.spec, c.conf, &IO{})
|
|
if err != nil {
|
|
log.G(ctx).WithError(err).WithField("id", c.id).Warn("could not create servicing container")
|
|
return
|
|
}
|
|
defer serviceCtr.Close()
|
|
|
|
err = serviceCtr.Start(ctx)
|
|
if err != nil {
|
|
log.G(ctx).WithError(err).WithField("id", c.id).Warn("failed to start servicing container")
|
|
serviceCtr.Terminate()
|
|
return
|
|
}
|
|
|
|
err = serviceCtr.processes[0].Wait()
|
|
if err == nil {
|
|
_, err = serviceCtr.processes[0].ExitCode()
|
|
log.G(ctx).WithError(err).WithField("id", c.id).Errorf("failed to retrieve servicing container exit code")
|
|
}
|
|
|
|
if err != nil {
|
|
if err := serviceCtr.Terminate(); err != nil {
|
|
log.G(ctx).WithError(err).WithField("id", c.id).Errorf("failed to terminate servicing container")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *Container) ExitCode() (uint32, error) {
|
|
if len(c.processes) == 0 {
|
|
return 255, errors.New("container not started")
|
|
}
|
|
return c.processes[0].ExitCode()
|
|
}
|
|
|
|
func (c *Container) GetConfiguration() Configuration {
|
|
return c.conf
|
|
}
|
|
|
|
func (c *Container) AddProcess(ctx context.Context, spec specs.Process, io *IO) (*Process, error) {
|
|
if len(c.processes) == 0 {
|
|
return nil, errors.New("container not started")
|
|
}
|
|
|
|
return c.addProcess(ctx, spec, io)
|
|
}
|
|
|
|
func (c *Container) addProcess(ctx context.Context, spec specs.Process, pio *IO) (*Process, error) {
|
|
// If we don't have a process yet, reused the container pid
|
|
var pid uint32
|
|
if len(c.processes) == 0 {
|
|
pid = c.pid
|
|
} else {
|
|
pid, err := c.hcs.pidPool.Get()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
if err != nil {
|
|
c.hcs.pidPool.Put(pid)
|
|
}
|
|
}()
|
|
}
|
|
|
|
conf := hcsshim.ProcessConfig{
|
|
EmulateConsole: pio.terminal,
|
|
CreateStdInPipe: pio.stdin != nil,
|
|
CreateStdOutPipe: pio.stdout != nil,
|
|
CreateStdErrPipe: pio.stderr != nil,
|
|
User: spec.User.Username,
|
|
CommandLine: strings.Join(spec.Args, " "),
|
|
Environment: ociSpecEnvToHCSEnv(spec.Env),
|
|
WorkingDirectory: spec.Cwd,
|
|
ConsoleSize: [2]uint{spec.ConsoleSize.Height, spec.ConsoleSize.Width},
|
|
}
|
|
|
|
if conf.WorkingDirectory == "" {
|
|
conf.WorkingDirectory = c.spec.Process.Cwd
|
|
}
|
|
|
|
proc, err := c.CreateProcess(&conf)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to create process")
|
|
}
|
|
|
|
stdin, stdout, stderr, err := proc.Stdio()
|
|
if err != nil {
|
|
proc.Kill()
|
|
return nil, errors.Wrapf(err, "failed to retrieve process stdio")
|
|
}
|
|
|
|
if pio.stdin != nil {
|
|
go func() {
|
|
log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdin: copy started")
|
|
io.Copy(stdin, pio.stdin)
|
|
log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdin: copy done")
|
|
stdin.Close()
|
|
pio.stdin.Close()
|
|
}()
|
|
} else {
|
|
proc.CloseStdin()
|
|
}
|
|
|
|
if pio.stdout != nil {
|
|
go func() {
|
|
log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdout: copy started")
|
|
io.Copy(pio.stdout, stdout)
|
|
log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stdout: copy done")
|
|
stdout.Close()
|
|
pio.stdout.Close()
|
|
}()
|
|
}
|
|
|
|
if pio.stderr != nil {
|
|
go func() {
|
|
log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stderr: copy started")
|
|
io.Copy(pio.stderr, stderr)
|
|
log.G(ctx).WithFields(logrus.Fields{"id": c.id, "pid": pid}).Debug("stderr: copy done")
|
|
stderr.Close()
|
|
pio.stderr.Close()
|
|
}()
|
|
}
|
|
|
|
p := &Process{
|
|
Process: proc,
|
|
pid: pid,
|
|
io: pio,
|
|
ecSync: make(chan struct{}),
|
|
}
|
|
|
|
c.Lock()
|
|
c.processes = append(c.processes, p)
|
|
idx := len(c.processes) - 1
|
|
c.Unlock()
|
|
|
|
go func() {
|
|
p.ec, p.ecErr = processExitCode(c.ID(), p)
|
|
close(p.ecSync)
|
|
c.Lock()
|
|
p.Delete()
|
|
// Remove process from slice (but keep the init one around)
|
|
if idx > 0 {
|
|
c.processes[idx] = c.processes[len(c.processes)-1]
|
|
c.processes[len(c.processes)-1] = nil
|
|
c.processes = c.processes[:len(c.processes)-1]
|
|
}
|
|
c.Unlock()
|
|
}()
|
|
|
|
return p, nil
|
|
}
|
|
|
|
// newHCSConfiguration generates a hcsshim configuration from the instance
|
|
// OCI Spec and hcs.Configuration.
|
|
func newContainerConfig(owner, id string, spec specs.Spec, conf Configuration) (*hcsshim.ContainerConfig, error) {
|
|
configuration := &hcsshim.ContainerConfig{
|
|
SystemType: "Container",
|
|
Name: id,
|
|
Owner: owner,
|
|
HostName: spec.Hostname,
|
|
IgnoreFlushesDuringBoot: conf.IgnoreFlushesDuringBoot,
|
|
HvPartition: conf.UseHyperV,
|
|
AllowUnqualifiedDNSQuery: conf.AllowUnqualifiedDNSQuery,
|
|
EndpointList: conf.NetworkEndpoints,
|
|
NetworkSharedContainerName: conf.NetworkSharedContainerID,
|
|
Credentials: conf.Credentials,
|
|
}
|
|
|
|
// TODO: use the create request Mount for those
|
|
for _, layerPath := range conf.Layers {
|
|
_, filename := filepath.Split(layerPath)
|
|
guid, err := hcsshim.NameToGuid(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
configuration.Layers = append(configuration.Layers, hcsshim.Layer{
|
|
ID: guid.ToString(),
|
|
Path: layerPath,
|
|
})
|
|
}
|
|
|
|
if len(spec.Mounts) > 0 {
|
|
mds := make([]hcsshim.MappedDir, len(spec.Mounts))
|
|
for i, mount := range spec.Mounts {
|
|
mds[i] = hcsshim.MappedDir{
|
|
HostPath: mount.Source,
|
|
ContainerPath: mount.Destination,
|
|
ReadOnly: false,
|
|
}
|
|
for _, o := range mount.Options {
|
|
if strings.ToLower(o) == "ro" {
|
|
mds[i].ReadOnly = true
|
|
}
|
|
}
|
|
}
|
|
configuration.MappedDirectories = mds
|
|
}
|
|
|
|
if conf.DNSSearchList != nil {
|
|
configuration.DNSSearchList = strings.Join(conf.DNSSearchList, ",")
|
|
}
|
|
|
|
if configuration.HvPartition {
|
|
for _, layerPath := range conf.Layers {
|
|
utilityVMPath := filepath.Join(layerPath, "UtilityVM")
|
|
_, err := os.Stat(utilityVMPath)
|
|
if err == nil {
|
|
configuration.HvRuntime = &hcsshim.HvRuntime{ImagePath: utilityVMPath}
|
|
break
|
|
} else if !os.IsNotExist(err) {
|
|
return nil, errors.Wrapf(err, "failed to access layer %s", layerPath)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(configuration.Layers) == 0 {
|
|
// TODO: support starting with 0 layers, this mean we need the "filter" directory as parameter
|
|
return nil, errors.New("at least one layers must be provided")
|
|
}
|
|
|
|
di := hcsshim.DriverInfo{
|
|
Flavour: 1, // filter driver
|
|
}
|
|
|
|
if len(configuration.Layers) > 0 {
|
|
di.HomeDir = filepath.Dir(conf.Layers[0])
|
|
}
|
|
|
|
// Windows doesn't support creating a container with a readonly
|
|
// filesystem, so always create a RW one
|
|
if err := hcsshim.CreateSandboxLayer(di, id, conf.Layers[0], conf.Layers); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to create sandbox layer for %s: layers: %#v, driverInfo: %#v",
|
|
id, configuration.Layers, di)
|
|
}
|
|
configuration.LayerFolderPath = filepath.Join(di.HomeDir, id)
|
|
|
|
err := hcsshim.ActivateLayer(di, id)
|
|
if err != nil {
|
|
removeLayer(context.TODO(), configuration.LayerFolderPath)
|
|
return nil, errors.Wrapf(err, "failed to active layer %s", configuration.LayerFolderPath)
|
|
}
|
|
|
|
err = hcsshim.PrepareLayer(di, id, conf.Layers)
|
|
if err != nil {
|
|
removeLayer(context.TODO(), configuration.LayerFolderPath)
|
|
return nil, errors.Wrapf(err, "failed to prepare layer %s", configuration.LayerFolderPath)
|
|
}
|
|
|
|
volumePath, err := hcsshim.GetLayerMountPath(di, id)
|
|
if err != nil {
|
|
if err := hcsshim.DestroyLayer(di, id); err != nil {
|
|
log.L.Warnf("failed to DestroyLayer %s: %s", id, err)
|
|
}
|
|
return nil, errors.Wrapf(err, "failed to getmount path for layer %s: driverInfo: %#v", id, di)
|
|
}
|
|
configuration.VolumePath = volumePath
|
|
|
|
return configuration, nil
|
|
}
|
|
|
|
// removeLayer deletes the given layer, all associated containers must have
|
|
// been shutdown for this to succeed.
|
|
func removeLayer(ctx context.Context, path string) error {
|
|
layerID := filepath.Base(path)
|
|
parentPath := filepath.Dir(path)
|
|
di := hcsshim.DriverInfo{
|
|
Flavour: 1, // filter driver
|
|
HomeDir: parentPath,
|
|
}
|
|
|
|
err := hcsshim.UnprepareLayer(di, layerID)
|
|
if err != nil {
|
|
log.G(ctx).WithError(err).Warnf("failed to unprepare layer %s for removal", path)
|
|
}
|
|
|
|
err = hcsshim.DeactivateLayer(di, layerID)
|
|
if err != nil {
|
|
log.G(ctx).WithError(err).Warnf("failed to deactivate layer %s for removal", path)
|
|
}
|
|
|
|
removePath := filepath.Join(parentPath, fmt.Sprintf("%s-removing", layerID))
|
|
err = os.Rename(path, removePath)
|
|
if err != nil {
|
|
log.G(ctx).WithError(err).Warnf("failed to rename container layer %s for removal", path)
|
|
removePath = path
|
|
}
|
|
if err := hcsshim.DestroyLayer(di, removePath); err != nil {
|
|
log.G(ctx).WithError(err).Errorf("failed to remove container layer %s", removePath)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ociSpecEnvToHCSEnv converts from the OCI Spec ENV format to the one
|
|
// expected by HCS.
|
|
func ociSpecEnvToHCSEnv(a []string) map[string]string {
|
|
env := make(map[string]string)
|
|
for _, s := range a {
|
|
arr := strings.SplitN(s, "=", 2)
|
|
if len(arr) == 2 {
|
|
env[arr[0]] = arr[1]
|
|
}
|
|
}
|
|
return env
|
|
}
|