Merge pull request #2670 from jterry75/runhcs_stop_success

runhcs-shim improvements and fixes to ctr

cmd/ctr/commands/run/run.go

@@ -106,6 +106,10 @@ var Command = cli.Command{
 			Name:  "fifo-dir",
 			Usage: "directory used for storing IO FIFOs",
 		},
+		cli.BoolFlag{
+			Name:  "isolated",
+			Usage: "run the container with vm isolation",
+		},
 	}, append(commands.SnapshotterFlags, commands.ContainerFlags...)...),
 	Action: func(context *cli.Context) error {
 		var (

cmd/ctr/commands/run/run_windows.go

@@ -31,57 +31,80 @@ import (
 // NewContainer creates a new container
 func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli.Context) (containerd.Container, error) {
 	var (
-		ref  = context.Args().First()
+		id    string
-		id   = context.Args().Get(1)
-		args = context.Args()[2:]
-	)
-
-	image, err := client.GetImage(ctx, ref)
-	if err != nil {
-		return nil, err
-	}
-
-	var (
 		opts  []oci.SpecOpts
 		cOpts []containerd.NewContainerOpts
 		spec  containerd.NewContainerOpts
+
+		config = context.IsSet("config")
 	)
 
-	if context.IsSet("config") {
+	if config {
+		id = context.Args().First()
 		opts = append(opts, oci.WithSpecFromFile(context.String("config")))
 	} else {
-		opts = append(opts, oci.WithDefaultSpec())
+		var (
-	}
+			ref  = context.Args().First()
+			args = context.Args()[2:]
+		)
 
-	opts = append(opts, oci.WithImageConfig(image))
+		id = context.Args().Get(1)
-	opts = append(opts, oci.WithEnv(context.StringSlice("env")))
+		snapshotter := context.String("snapshotter")
-	opts = append(opts, withMounts(context))
+		if snapshotter == "windows-lcow" {
-	if context.Bool("tty") {
+			opts = append(opts, oci.WithDefaultSpecForPlatform("linux/amd64"))
-		opts = append(opts, oci.WithTTY)
+			// Clear the rootfs section.
-
+			opts = append(opts, oci.WithRootFSPath(""))
-		con := console.Current()
+		} else {
-		size, err := con.Size()
+			opts = append(opts, oci.WithDefaultSpec())
-		if err != nil {
+		}
-			logrus.WithError(err).Error("console size")
+		opts = append(opts, oci.WithEnv(context.StringSlice("env")))
+		opts = append(opts, withMounts(context))
+
+		image, err := client.GetImage(ctx, ref)
+		if err != nil {
+			return nil, err
+		}
+		unpacked, err := image.IsUnpacked(ctx, snapshotter)
+		if err != nil {
+			return nil, err
+		}
+		if !unpacked {
+			if err := image.Unpack(ctx, snapshotter); err != nil {
+				return nil, err
+			}
+		}
+		opts = append(opts, oci.WithImageConfig(image))
+		cOpts = append(cOpts, containerd.WithImage(image))
+		cOpts = append(cOpts, containerd.WithSnapshotter(snapshotter))
+		cOpts = append(cOpts, containerd.WithNewSnapshot(id, image))
+
+		if len(args) > 0 {
+			opts = append(opts, oci.WithProcessArgs(args...))
+		}
+		if cwd := context.String("cwd"); cwd != "" {
+			opts = append(opts, oci.WithProcessCwd(cwd))
+		}
+		if context.Bool("tty") {
+			opts = append(opts, oci.WithTTY)
+
+			con := console.Current()
+			size, err := con.Size()
+			if err != nil {
+				logrus.WithError(err).Error("console size")
+			}
+			opts = append(opts, oci.WithTTYSize(int(size.Width), int(size.Height)))
+		}
+		if context.Bool("isolated") {
+			opts = append(opts, oci.WithWindowsHyperV)
 		}
-		opts = append(opts, oci.WithTTYSize(int(size.Width), int(size.Height)))
 	}
 
-	if len(args) > 0 {
+	cOpts = append(cOpts, containerd.WithContainerLabels(commands.LabelArgs(context.StringSlice("label"))))
-		opts = append(opts, oci.WithProcessArgs(args...))
+	cOpts = append(cOpts, containerd.WithRuntime(context.String("runtime"), nil))
-	}
-	if cwd := context.String("cwd"); cwd != "" {
-		opts = append(opts, oci.WithProcessCwd(cwd))
-	}
 
 	var s specs.Spec
 	spec = containerd.WithSpec(&s, opts...)
 
-	cOpts = append(cOpts, containerd.WithContainerLabels(commands.LabelArgs(context.StringSlice("label"))))
-	cOpts = append(cOpts, containerd.WithImage(image))
-	cOpts = append(cOpts, containerd.WithSnapshotter(context.String("snapshotter")))
-	cOpts = append(cOpts, containerd.WithNewSnapshot(id, image))
-	cOpts = append(cOpts, containerd.WithRuntime(context.String("runtime"), nil))
 	cOpts = append(cOpts, spec)
 
 	return client.NewContainer(ctx, id, cOpts...)

oci/spec_opts.go

@@ -1011,3 +1011,14 @@ var WithPrivileged = Compose(
 	WithApparmorProfile(""),
 	WithSeccompUnconfined,
 )
+
+// WithWindowsHyperV sets the Windows.HyperV section for HyperV isolation of containers.
+func WithWindowsHyperV(_ context.Context, _ Client, _ *containers.Container, s *Spec) error {
+	if s.Windows == nil {
+		s.Windows = &specs.Windows{}
+	}
+	if s.Windows.HyperV == nil {
+		s.Windows.HyperV = &specs.WindowsHyperV{}
+	}
+	return nil
+}

runtime/v2/runhcs/cmdutil.go

@@ -1,98 +0,0 @@
-// +build windows
-
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package runhcs
-
-import (
-	"bytes"
-	"context"
-	"os/exec"
-	"sync"
-	"syscall"
-	"time"
-)
-
-var (
-	bytesBufferPool = sync.Pool{
-		New: func() interface{} {
-			return bytes.NewBuffer(nil)
-		},
-	}
-)
-
-func getBuffer() *bytes.Buffer {
-	return bytesBufferPool.Get().(*bytes.Buffer)
-}
-
-func putBuffer(b *bytes.Buffer) {
-	b.Reset()
-	bytesBufferPool.Put(b)
-}
-
-type processExit struct {
-	pid        uint32
-	exitStatus uint32
-	exitedAt   time.Time
-	exitErr    error
-}
-
-func runCmd(ctx context.Context, c *exec.Cmd) (*processExit, error) {
-	ec, startErr := startCmd(ctx, c)
-	if startErr != nil {
-		return nil, startErr
-	}
-	er, cmdErr := waitCmd(ctx, ec)
-	return er, cmdErr
-}
-
-func startCmd(ctx context.Context, c *exec.Cmd) (<-chan *processExit, error) {
-	if err := c.Start(); err != nil {
-		return nil, err
-	}
-	ec := make(chan *processExit, 1)
-	go func() {
-		defer close(ec)
-
-		var status int
-		eerr := c.Wait()
-		if eerr != nil {
-			status = 255
-			if exitErr, ok := eerr.(*exec.ExitError); ok {
-				if ws, ok := exitErr.Sys().(syscall.WaitStatus); ok {
-					status = ws.ExitStatus()
-				}
-			}
-		}
-		ec <- &processExit{
-			pid:        uint32(c.Process.Pid),
-			exitStatus: uint32(status),
-			exitedAt:   time.Now(),
-			exitErr:    eerr,
-		}
-	}()
-
-	return ec, nil
-}
-
-func waitCmd(ctx context.Context, ec <-chan *processExit) (*processExit, error) {
-	e := <-ec
-	if e.exitStatus != 0 {
-		return e, e.exitErr
-	}
-	return e, nil
-}

runtime/v2/runhcs/process.go

@@ -31,6 +31,13 @@ import (
 	"github.com/containerd/containerd/runtime"
 )
 
+type processExit struct {
+	pid        uint32
+	exitStatus uint32
+	exitedAt   time.Time
+	exitErr    error
+}
+
 func newProcess(ctx context.Context, s *service, id string, pid uint32, pr *pipeRelay, bundle, stdin, stdout, stderr string, terminal bool) (*process, error) {
 	p, err := os.FindProcess(int(pid))
 	if err != nil {

runtime/v2/runhcs/service.go

@@ -49,9 +49,7 @@ import (
 )
 
 const (
-	runhcsBinary      = "runhcs"
+	runhcsShimVersion = "0.0.1"
-	runhcsVersion     = "0.0.1"
-	runhcsDebugLegacy = "--debug" // TODO: JTERRY75 remove when all cmd's are complete in go-runhcs
 )
 
 var (
@@ -201,41 +199,9 @@ func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (*taskAPI.
 		return nil, err
 	}
 
-	cmd := exec.Command(runhcsBinary, runhcsDebugLegacy, "state", p.id)
+	rhcs := newRunhcs(p.bundle)
-	sout := getBuffer()
+	cs, err := rhcs.State(ctx, p.id)
-	defer putBuffer(sout)
+	if err != nil {
-
-	cmd.Stdout = sout
-	_, stateErr := runCmd(ctx, cmd)
-	if stateErr != nil {
-		return nil, stateErr
-	}
-
-	// TODO: JTERRY75 merge this with runhcs declaration
-	type containerState struct {
-		// Version is the OCI version for the container
-		Version string `json:"ociVersion"`
-		// ID is the container ID
-		ID string `json:"id"`
-		// InitProcessPid is the init process id in the parent namespace
-		InitProcessPid int `json:"pid"`
-		// Status is the current status of the container, running, paused, ...
-		Status string `json:"status"`
-		// Bundle is the path on the filesystem to the bundle
-		Bundle string `json:"bundle"`
-		// Rootfs is a path to a directory containing the container's root filesystem.
-		Rootfs string `json:"rootfs"`
-		// Created is the unix timestamp for the creation time of the container in UTC
-		Created time.Time `json:"created"`
-		// Annotations is the user defined annotations added to the config.
-		Annotations map[string]string `json:"annotations,omitempty"`
-		// The owner of the state directory (the owner of the container).
-		Owner string `json:"owner"`
-	}
-
-	var cs containerState
-	if err := json.NewDecoder(sout).Decode(&cs); err != nil {
-		log.G(ctx).WithError(err).Debugf("failed to decode runhcs state output: %s", sout.Bytes())
 		return nil, err
 	}
 
@@ -551,13 +517,14 @@ func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (*taskAPI.Pi
 // Pause the container
 func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (*ptypes.Empty, error) {
 	// TODO: Validate that 'id' is actually a valid parent container ID
-	if _, err := s.getProcess(r.ID, ""); err != nil {
+	var p *process
+	var err error
+	if p, err = s.getProcess(r.ID, ""); err != nil {
 		return nil, err
 	}
 
-	cmd := exec.Command(runhcsBinary, runhcsDebugLegacy, "pause", r.ID)
+	rhcs := newRunhcs(p.bundle)
-	_, err := runCmd(ctx, cmd)
+	if err = rhcs.Pause(ctx, p.id); err != nil {
-	if err != nil {
 		return nil, err
 	}
 
@@ -567,13 +534,14 @@ func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (*ptypes.E
 // Resume the container
 func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (*ptypes.Empty, error) {
 	// TODO: Validate that 'id' is actually a valid parent container ID
-	if _, err := s.getProcess(r.ID, ""); err != nil {
+	var p *process
+	var err error
+	if p, err = s.getProcess(r.ID, ""); err != nil {
 		return nil, err
 	}
 
-	cmd := exec.Command(runhcsBinary, runhcsDebugLegacy, "resume", r.ID)
+	rhcs := newRunhcs(p.bundle)
-	_, err := runCmd(ctx, cmd)
+	if err = rhcs.Resume(ctx, p.id); err != nil {
-	if err != nil {
 		return nil, err
 	}
 
@@ -599,7 +567,7 @@ func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (*ptypes.Emp
 	// TODO: JTERRY75 runhcs support for r.All?
 	rhcs := newRunhcs(p.bundle)
 	if err = rhcs.Kill(ctx, p.id, strconv.FormatUint(uint64(r.Signal), 10)); err != nil {
-		if !strings.Contains(err.Error(), "container is not running") {
+		if !strings.Contains(err.Error(), "container is stopped") {
 			return nil, err
 		}
 	}
@@ -698,18 +666,12 @@ func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (*
 		return nil, err
 	}
 
-	cmd := exec.Command(
+	pid := int(p.pid)
-		runhcsBinary,
+	opts := runhcs.ResizeTTYOpts{
-		runhcsDebugLegacy,
+		Pid: &pid,
-		"resize-tty",
+	}
-		p.cid,
+	rhcs := newRunhcs(p.bundle)
-		"-p",
+	if err = rhcs.ResizeTTY(ctx, p.cid, uint16(r.Width), uint16(r.Height), &opts); err != nil {
-		strconv.FormatUint(uint64(p.pid), 10),
-		strconv.FormatUint(uint64(r.Width), 10),
-		strconv.FormatUint(uint64(r.Height), 10))
-
-	_, err = runCmd(ctx, cmd)
-	if err != nil {
 		return nil, err
 	}
 
@@ -756,7 +718,7 @@ func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (*task
 	return &taskAPI.ConnectResponse{
 		ShimPid: uint32(os.Getpid()),
 		TaskPid: s.processes[s.id].pid,
-		Version: runhcsVersion,
+		Version: runhcsShimVersion,
 	}, nil
 }
 

snapshots/lcow/lcow.go

@@ -23,13 +23,13 @@ import (
 	"encoding/json"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"syscall"
 	"time"
 	"unsafe"
 
+	"github.com/Microsoft/hcsshim/cmd/go-runhcs"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/mount"
@@ -337,16 +337,19 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
 						continue
 					}
 					defer slock.Close()
-					// Create the scratch
-					cmd := exec.Command(
-						"runhcs.exe",
-						"create-scratch",
-						"--destpath", scratchPath)
 
-					if bytes, err := cmd.CombinedOutput(); err != nil {
+					// Create the scratch
-						_ = os.Remove(scratchPath)
+					rhcs := runhcs.Runhcs{
-						return nil, errors.Wrapf(err, "failed to create scratch.vhdx. additional info: '%s'", string(bytes))
+						Debug:     true,
+						Log:       filepath.Join(s.root, "runhcs-scratch.log"),
+						LogFormat: runhcs.JSON,
+						Owner:     "containerd",
 					}
+					if err := rhcs.CreateScratch(ctx, scratchPath); err != nil {
+						_ = os.Remove(scratchPath)
+						return nil, errors.Wrap(err, "failed to create scratch.vhdx")
+					}
+
 					// Successfully created scratch in the cache. Open and copy
 					continue
 				} else {

vendor.conf

@@ -33,7 +33,7 @@ golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0
 github.com/Microsoft/go-winio v0.4.10
-github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55
+github.com/Microsoft/hcsshim v0.7.4
 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4
 github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a

vendor/github.com/Microsoft/hcsshim/cmd/go-runhcs/runhcs_create-scratch.go

@@ -0,0 +1,10 @@
+package runhcs
+
+import (
+	"context"
+)
+
+// CreateScratch creates a scratch vhdx at 'destpath' that is ext4 formatted.
+func (r *Runhcs) CreateScratch(context context.Context, destpath string) error {
+	return r.runOrError(r.command(context, "create-scratch", "--destpath", destpath))
+}

vendor/github.com/Microsoft/hcsshim/cmd/go-runhcs/runhcs_list.go

@@ -0,0 +1,28 @@
+package runhcs
+
+import (
+	"context"
+	"encoding/json"
+
+	irunhcs "github.com/Microsoft/hcsshim/internal/runhcs"
+)
+
+// ContainerState is the representation of the containers state at the moment of
+// query.
+type ContainerState = irunhcs.ContainerState
+
+// List containers started by runhcs.
+//
+// Note: This is specific to the Runhcs.Root namespace provided in the global
+// settings.
+func (r *Runhcs) List(context context.Context) ([]*ContainerState, error) {
+	data, err := cmdOutput(r.command(context, "list", "--format=json"), false)
+	if err != nil {
+		return nil, err
+	}
+	var out []*ContainerState
+	if err := json.Unmarshal(data, &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}

vendor/github.com/Microsoft/hcsshim/cmd/go-runhcs/runhcs_pause.go

@@ -0,0 +1,10 @@
+package runhcs
+
+import (
+	"context"
+)
+
+// Pause suspends all processes inside the container.
+func (r *Runhcs) Pause(context context.Context, id string) error {
+	return r.runOrError(r.command(context, "pause", id))
+}

vendor/github.com/Microsoft/hcsshim/cmd/go-runhcs/runhcs_ps.go

@@ -0,0 +1,20 @@
+package runhcs
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+)
+
+// Ps displays the processes running inside a container.
+func (r *Runhcs) Ps(context context.Context, id string) ([]int, error) {
+	data, err := cmdOutput(r.command(context, "ps", "--format=json", id), true)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %s", err, data)
+	}
+	var out []int
+	if err := json.Unmarshal(data, &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}

vendor/github.com/Microsoft/hcsshim/cmd/go-runhcs/runhcs_resize-tty.go

@@ -0,0 +1,33 @@
+package runhcs
+
+import (
+	"context"
+	"strconv"
+)
+
+// ResizeTTYOpts is set of options that can be used with the ResizeTTY command.
+type ResizeTTYOpts struct {
+	// Pid is the process pid (defaults to init pid).
+	Pid *int
+}
+
+func (opt *ResizeTTYOpts) args() ([]string, error) {
+	var out []string
+	if opt.Pid != nil {
+		out = append(out, "--pid", strconv.Itoa(*opt.Pid))
+	}
+	return out, nil
+}
+
+// ResizeTTY updates the terminal size for a container process.
+func (r *Runhcs) ResizeTTY(context context.Context, id string, width, height uint16, opts *ResizeTTYOpts) error {
+	args := []string{"resize-tty"}
+	if opts != nil {
+		oargs, err := opts.args()
+		if err != nil {
+			return err
+		}
+		args = append(args, oargs...)
+	}
+	return r.runOrError(r.command(context, append(args, id, strconv.FormatUint(uint64(width), 10), strconv.FormatUint(uint64(height), 10))...))
+}

vendor/github.com/Microsoft/hcsshim/cmd/go-runhcs/runhcs_resume.go

@@ -0,0 +1,10 @@
+package runhcs
+
+import (
+	"context"
+)
+
+// Resume resumes all processes that have been previously paused.
+func (r *Runhcs) Resume(context context.Context, id string) error {
+	return r.runOrError(r.command(context, "resume", id))
+}

vendor/github.com/Microsoft/hcsshim/cmd/go-runhcs/runhcs_state.go

@@ -0,0 +1,20 @@
+package runhcs
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+)
+
+// State outputs the state of a container.
+func (r *Runhcs) State(context context.Context, id string) (*ContainerState, error) {
+	data, err := cmdOutput(r.command(context, "state", id), true)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %s", err, data)
+	}
+	var out ContainerState
+	if err := json.Unmarshal(data, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}

vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go

@@ -2,10 +2,16 @@ package guid
 
 import (
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io"
+	"strconv"
+	"strings"
 )
 
+var _ = (json.Marshaler)(&GUID{})
+var _ = (json.Unmarshaler)(&GUID{})
+
 type GUID [16]byte
 
 func New() GUID {
@@ -20,3 +26,44 @@ func New() GUID {
 func (g GUID) String() string {
 	return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:])
 }
+
+func FromString(s string) GUID {
+	if len(s) != 36 {
+		panic(fmt.Sprintf("invalid GUID length: %d", len(s)))
+	}
+	if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' {
+		panic("invalid GUID format")
+	}
+	indexOrder := [16]int{
+		0, 2, 4, 6,
+		9, 11,
+		14, 16,
+		19, 21,
+		24, 26, 28, 30, 32, 34,
+	}
+	byteOrder := [16]int{
+		3, 2, 1, 0,
+		5, 4,
+		7, 6,
+		8, 9,
+		10, 11, 12, 13, 14, 15,
+	}
+	var g GUID
+	for i, x := range indexOrder {
+		b, err := strconv.ParseInt(s[x:x+2], 16, 16)
+		if err != nil {
+			panic(err)
+		}
+		g[byteOrder[i]] = byte(b)
+	}
+	return g
+}
+
+func (g GUID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(g.String())
+}
+
+func (g *GUID) UnmarshalJSON(data []byte) error {
+	*g = FromString(strings.Trim(string(data), "\""))
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go

@@ -2,6 +2,7 @@ package hcs
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"sync"
 	"syscall"
@@ -83,7 +84,10 @@ func (process *Process) Kill() error {
 	}
 
 	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("TerminateProcess %s: %d", process.SystemID(), process.Pid()), &completed)
 	err := hcsTerminateProcess(process.handle, &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return makeProcessError(process, operation, err, events)
@@ -177,7 +181,10 @@ func (process *Process) Properties() (*ProcessStatus, error) {
 		resultp     *uint16
 		propertiesp *uint16
 	)
+	completed := false
+	go syscallWatcher(fmt.Sprintf("GetProcessProperties %s: %d", process.SystemID(), process.Pid()), &completed)
 	err := hcsGetProcessProperties(process.handle, &propertiesp, &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return nil, makeProcessError(process, operation, err, events)

vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go

@@ -2,6 +2,7 @@ package hcs
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"strconv"
 	"sync"
@@ -63,7 +64,10 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System,
 		resultp  *uint16
 		identity syscall.Handle
 	)
+	completed := false
+	go syscallWatcher(fmt.Sprintf("CreateCompleteSystem %s: %s", id, hcsDocument), &completed)
 	createError := hcsCreateComputeSystem(id, hcsDocument, identity, &computeSystem.handle, &resultp)
+	completed = true
 
 	if createError == nil || IsPending(createError) {
 		if err := computeSystem.registerCallback(); err != nil {
@@ -74,7 +78,7 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System,
 		}
 	}
 
-	events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.Duration)
+	events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.SystemCreate)
 	if err != nil {
 		if err == ErrTimeout {
 			// Terminate the compute system if it still exists. We're okay to
@@ -135,7 +139,10 @@ func GetComputeSystems(q schema1.ComputeSystemQuery) ([]schema1.ContainerPropert
 		resultp         *uint16
 		computeSystemsp *uint16
 	)
+	completed := false
+	go syscallWatcher(fmt.Sprintf("GetComputeSystems %s:", query), &completed)
 	err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return nil, &HcsError{Op: operation, Err: err, Events: events}
@@ -192,8 +199,11 @@ func (computeSystem *System) Start() error {
 	}
 
 	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("StartComputeSystem %s:", computeSystem.ID()), &completed)
 	err := hcsStartComputeSystem(computeSystem.handle, "", &resultp)
-	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.Duration)
+	completed = true
+	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart)
 	if err != nil {
 		return makeSystemError(computeSystem, "Start", "", err, events)
 	}
@@ -219,7 +229,10 @@ func (computeSystem *System) Shutdown() error {
 	}
 
 	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("ShutdownComputeSystem %s:", computeSystem.ID()), &completed)
 	err := hcsShutdownComputeSystem(computeSystem.handle, "", &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return makeSystemError(computeSystem, "Shutdown", "", err, events)
@@ -242,7 +255,10 @@ func (computeSystem *System) Terminate() error {
 	}
 
 	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("TerminateComputeSystem %s:", computeSystem.ID()), &completed)
 	err := hcsTerminateComputeSystem(computeSystem.handle, "", &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return makeSystemError(computeSystem, "Terminate", "", err, events)
@@ -291,7 +307,10 @@ func (computeSystem *System) Properties(types ...schema1.PropertyType) (*schema1
 	}
 
 	var resultp, propertiesp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("GetComputeSystemProperties %s:", computeSystem.ID()), &completed)
 	err = hcsGetComputeSystemProperties(computeSystem.handle, string(queryj), &propertiesp, &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return nil, makeSystemError(computeSystem, "Properties", "", err, events)
@@ -320,8 +339,11 @@ func (computeSystem *System) Pause() error {
 	}
 
 	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("PauseComputeSystem %s:", computeSystem.ID()), &completed)
 	err := hcsPauseComputeSystem(computeSystem.handle, "", &resultp)
-	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.Duration)
+	completed = true
+	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause)
 	if err != nil {
 		return makeSystemError(computeSystem, "Pause", "", err, events)
 	}
@@ -342,8 +364,11 @@ func (computeSystem *System) Resume() error {
 	}
 
 	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("ResumeComputeSystem %s:", computeSystem.ID()), &completed)
 	err := hcsResumeComputeSystem(computeSystem.handle, "", &resultp)
-	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.Duration)
+	completed = true
+	events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume)
 	if err != nil {
 		return makeSystemError(computeSystem, "Resume", "", err, events)
 	}
@@ -375,7 +400,10 @@ func (computeSystem *System) CreateProcess(c interface{}) (*Process, error) {
 	configuration := string(configurationb)
 	logrus.Debugf(title+" config=%s", configuration)
 
+	completed := false
+	go syscallWatcher(fmt.Sprintf("CreateProcess %s: %s", computeSystem.ID(), configuration), &completed)
 	err = hcsCreateProcess(computeSystem.handle, configuration, &processInfo, &processHandle, &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return nil, makeSystemError(computeSystem, "CreateProcess", configuration, err, events)
@@ -415,7 +443,10 @@ func (computeSystem *System) OpenProcess(pid int) (*Process, error) {
 		return nil, makeSystemError(computeSystem, "OpenProcess", "", ErrAlreadyClosed, nil)
 	}
 
+	completed := false
+	go syscallWatcher(fmt.Sprintf("OpenProcess %s: %d", computeSystem.ID(), pid), &completed)
 	err := hcsOpenProcess(computeSystem.handle, uint32(pid), &processHandle, &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return nil, makeSystemError(computeSystem, "OpenProcess", "", err, events)
@@ -451,7 +482,11 @@ func (computeSystem *System) Close() error {
 		return makeSystemError(computeSystem, "Close", "", err, nil)
 	}
 
-	if err := hcsCloseComputeSystem(computeSystem.handle); err != nil {
+	completed := false
+	go syscallWatcher(fmt.Sprintf("CloseComputeSystem %s:", computeSystem.ID()), &completed)
+	err := hcsCloseComputeSystem(computeSystem.handle)
+	completed = true
+	if err != nil {
 		return makeSystemError(computeSystem, "Close", "", err, nil)
 	}
 
@@ -537,7 +572,10 @@ func (computeSystem *System) Modify(config interface{}) error {
 	logrus.Debugf(title + " " + requestString)
 
 	var resultp *uint16
+	completed := false
+	go syscallWatcher(fmt.Sprintf("ModifyComputeSystem %s: %s", computeSystem.ID(), requestString), &completed)
 	err = hcsModifyComputeSystem(computeSystem.handle, requestString, &resultp)
+	completed = true
 	events := processHcsResult(resultp)
 	if err != nil {
 		return makeSystemError(computeSystem, "Modify", requestString, err, events)

vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go

@@ -0,0 +1,30 @@
+package hcs
+
+import (
+	"time"
+
+	"github.com/Microsoft/hcsshim/internal/timeout"
+	"github.com/sirupsen/logrus"
+)
+
+// syscallWatcher is used as a very simple goroutine around calls into
+// the platform. In some cases, we have seen HCS APIs not returning due to
+// various bugs, and the goroutine making the syscall ends up not returning,
+// prior to its async callback. By spinning up a syscallWatcher, it allows
+// us to at least log a warning if a syscall doesn't complete in a reasonable
+// amount of time.
+//
+// Usage is:
+//
+// completed := false
+// go syscallWatcher("some description", &completed)
+// <syscall>
+// completed = true
+//
+func syscallWatcher(description string, syscallCompleted *bool) {
+	time.Sleep(timeout.SyscallWatcher)
+	if *syscallCompleted {
+		return
+	}
+	logrus.Warnf("%s: Did not complete within %s. This may indicate a platform issue. If it appears to be making no forward progress, obtain the stacks and see is there is a syscall stuck in the platform API for a significant length of time.", description, timeout.SyscallWatcher)
+}

vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go

@@ -1,4 +1,4 @@
-// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT
+// Code generated by 'go generate'; DO NOT EDIT.
 
 package interop
 

vendor/github.com/Microsoft/hcsshim/internal/runhcs/container.go

@@ -0,0 +1,26 @@
+package runhcs
+
+import "time"
+
+// ContainerState represents the platform agnostic pieces relating to a
+// running container's status and state
+type ContainerState struct {
+	// Version is the OCI version for the container
+	Version string `json:"ociVersion"`
+	// ID is the container ID
+	ID string `json:"id"`
+	// InitProcessPid is the init process id in the parent namespace
+	InitProcessPid int `json:"pid"`
+	// Status is the current status of the container, running, paused, ...
+	Status string `json:"status"`
+	// Bundle is the path on the filesystem to the bundle
+	Bundle string `json:"bundle"`
+	// Rootfs is a path to a directory containing the container's root filesystem.
+	Rootfs string `json:"rootfs"`
+	// Created is the unix timestamp for the creation time of the container in UTC
+	Created time.Time `json:"created"`
+	// Annotations is the user defined annotations added to the config.
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// The owner of the state directory (the owner of the container).
+	Owner string `json:"owner"`
+}

vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go

@@ -1,4 +1,4 @@
-// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT
+// Code generated by 'go generate'; DO NOT EDIT.
 
 package safefile
 

vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go

@@ -6,21 +6,65 @@ import (
 	"time"
 )
 
-// Duration is the default time to wait for various operations.
+var (
-// - Waiting for async notifications from HCS
+	// defaultTimeout is the timeout for most operations that is not overridden.
-// - Waiting for processes to launch through
+	defaultTimeout = 4 * time.Minute
-// - Waiting to copy data to/from a launched processes stdio pipes.
-//
-// This can be overridden through environment variable `HCS_TIMEOUT_SECONDS`
 
-var Duration = 4 * time.Minute
+	// defaultTimeoutTestdRetry is the retry loop timeout for testd to respond
+	// for a disk to come online in LCOW.
+	defaultTimeoutTestdRetry = 5 * time.Second
+)
+
+// External variables for HCSShim consumers to use.
+var (
+	// SystemCreate is the timeout for creating a compute system
+	SystemCreate time.Duration = defaultTimeout
+
+	// SystemStart is the timeout for starting a compute system
+	SystemStart time.Duration = defaultTimeout
+
+	// SystemPause is the timeout for pausing a compute system
+	SystemPause time.Duration = defaultTimeout
+
+	// SystemResume is the timeout for resuming a compute system
+	SystemResume time.Duration = defaultTimeout
+
+	// SyscallWatcher is the timeout before warning of a potential stuck platform syscall.
+	SyscallWatcher time.Duration = defaultTimeout
+
+	// Tar2VHD is the timeout for the tar2vhd operation to complete
+	Tar2VHD time.Duration = defaultTimeout
+
+	// ExternalCommandToStart is the timeout for external commands to start
+	ExternalCommandToStart = defaultTimeout
+
+	// ExternalCommandToComplete is the timeout for external commands to complete.
+	// Generally this means copying data from their stdio pipes.
+	ExternalCommandToComplete = defaultTimeout
+
+	// TestDRetryLoop is the timeout for testd retry loop when onlining a SCSI disk in LCOW
+	TestDRetryLoop = defaultTimeoutTestdRetry
+)
 
 func init() {
-	envTimeout := os.Getenv("HCSSHIM_TIMEOUT_SECONDS")
+	SystemCreate = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMCREATE", SystemCreate)
+	SystemStart = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMSTART", SystemStart)
+	SystemPause = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMPAUSE", SystemPause)
+	SystemResume = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMRESUME", SystemResume)
+	SyscallWatcher = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSCALLWATCHER", SyscallWatcher)
+	Tar2VHD = durationFromEnvironment("HCSSHIM_TIMEOUT_TAR2VHD", Tar2VHD)
+	ExternalCommandToStart = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDSTART", ExternalCommandToStart)
+	ExternalCommandToComplete = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDCOMPLETE", ExternalCommandToComplete)
+	TestDRetryLoop = durationFromEnvironment("HCSSHIM_TIMEOUT_TESTDRETRYLOOP", TestDRetryLoop)
+}
+
+func durationFromEnvironment(env string, defaultValue time.Duration) time.Duration {
+	envTimeout := os.Getenv(env)
 	if len(envTimeout) > 0 {
 		e, err := strconv.Atoi(envTimeout)
 		if err == nil && e > 0 {
-			Duration = time.Second * time.Duration(e)
+			return time.Second * time.Duration(e)
 		}
 	}
+	return defaultValue
 }

vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go

@@ -9,15 +9,15 @@ import (
 // the parent layer provided.
 func CreateLayer(path, parent string) error {
 	title := "hcsshim::CreateLayer "
-	logrus.Debugf(title+"Flavour %d ID %s parent %s", path, parent)
+	logrus.Debugf(title+"ID %s parent %s", path, parent)
 
 	err := createLayer(&stdDriverInfo, path, parent)
 	if err != nil {
-		err = hcserror.Errorf(err, title, "path=%s parent=%s flavour=%d", path, parent)
+		err = hcserror.Errorf(err, title, "path=%s parent=%s", path, parent)
 		logrus.Error(err)
 		return err
 	}
 
-	logrus.Debugf(title+" - succeeded path=%s parent=%s flavour=%d", path, parent)
+	logrus.Debugf(title+"- succeeded path=%s parent=%s", path, parent)
 	return nil
 }