Add container compute stats support.

Signed-off-by: Lantao Liu <lantaol@google.com>

pkg/server/container_stats_list_windows.go

@@ -19,7 +19,10 @@ limitations under the License.
 package server
 
 import (
+	wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
 	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 
 	containerstore "github.com/containerd/cri/pkg/store/container"
@@ -53,8 +56,29 @@ func (c *criService) containerMetrics(
 		Annotations: meta.Config.GetAnnotations(),
 	}
 
-	// TODO(windows): hcsshim Stats is not implemented, add windows support after
+	if stats != nil {
-	// that is implemented.
+		s, err := typeurl.UnmarshalAny(stats.Data)
-
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to extract container metrics")
+		}
+		wstats := s.(*wstats.Statistics).GetWindows()
+		if wstats == nil {
+			return nil, errors.New("windows stats is empty")
+		}
+		if wstats.Processor != nil {
+			cs.Cpu = &runtime.CpuUsage{
+				Timestamp:            wstats.Timestamp.UnixNano(),
+				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: wstats.Processor.TotalRuntimeNS},
+			}
+		}
+		if wstats.Memory != nil {
+			cs.Memory = &runtime.MemoryUsage{
+				Timestamp: wstats.Timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{
+					Value: wstats.Memory.MemoryUsagePrivateWorkingSetBytes,
+				},
+			}
+		}
+	}
 	return &cs, nil
 }

vendor.conf

@@ -48,7 +48,7 @@ github.com/containerd/containerd a6a0c8b6e36415a151d93d096c1c0af9e0bd7977
 github.com/containerd/console 0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f
 github.com/containerd/cgroups abd0b19954a6b05e0963f48427062d1481b7faad
 github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
-github.com/Microsoft/hcsshim c088f411aaf3585d8dffc9deb4289ffa32854497 # TODO(windows): update this in containerd/containerd
+github.com/Microsoft/hcsshim d2849cbdb9dfe5f513292a9610ca2eb734cdd1e7
 github.com/Microsoft/go-winio v0.4.14
 github.com/BurntSushi/toml v0.3.1
 

vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats/doc.go

@@ -0,0 +1,6 @@
+package stats
+
+import (
+	// go mod will not vendor without an import for metrics.proto
+	_ "github.com/containerd/cgroups/stats/v1"
+)

vendor/github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats/stats.proto

@@ -0,0 +1,70 @@
+syntax = "proto3";
+
+package containerd.runhcs.stats.v1;
+
+import weak "gogoproto/gogo.proto";
+import "google/protobuf/timestamp.proto";
+import "github.com/containerd/cgroups/stats/v1/metrics.proto";
+
+option go_package = "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats;stats";
+
+message Statistics {
+	oneof container {
+		WindowsContainerStatistics windows = 1;
+		io.containerd.cgroups.v1.Metrics linux = 2;
+	}
+	VirtualMachineStatistics vm = 3 [(gogoproto.customname) = "VM"];
+}
+
+message WindowsContainerStatistics {
+	google.protobuf.Timestamp timestamp = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+	google.protobuf.Timestamp container_start_time = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+	uint64 uptime_ns = 3 [(gogoproto.customname) = "UptimeNS"];
+	WindowsContainerProcessorStatistics processor = 4;
+	WindowsContainerMemoryStatistics memory = 5;
+	WindowsContainerStorageStatistics storage = 6;
+}
+
+message WindowsContainerProcessorStatistics {
+	uint64 total_runtime_ns = 1 [(gogoproto.customname) = "TotalRuntimeNS"];
+	uint64 runtime_user_ns = 2 [(gogoproto.customname) = "RuntimeUserNS"];
+	uint64 runtime_kernel_ns = 3 [(gogoproto.customname) = "RuntimeKernelNS"];
+}
+
+message WindowsContainerMemoryStatistics {
+	uint64 memory_usage_commit_bytes = 1;
+	uint64 memory_usage_commit_peak_bytes = 2;
+	uint64 memory_usage_private_working_set_bytes = 3;
+}
+
+message WindowsContainerStorageStatistics {
+	uint64 read_count_normalized = 1;
+	uint64 read_size_bytes = 2;
+	uint64 write_count_normalized = 3;
+	uint64 write_size_bytes = 4;
+}
+
+message VirtualMachineStatistics {
+	VirtualMachineProcessorStatistics processor = 1;
+	VirtualMachineMemoryStatistics memory = 2;
+}
+
+message VirtualMachineProcessorStatistics {
+	uint64 total_runtime_ns = 1 [(gogoproto.customname) = "TotalRuntimeNS"];
+}
+
+message VirtualMachineMemoryStatistics {
+	uint64 working_set_bytes = 1;
+	uint32 virtual_node_count = 2;
+	VirtualMachineMemory vm_memory = 3;
+}
+
+message VirtualMachineMemory {
+	int32 available_memory = 1;
+	int32 available_memory_buffer = 2;
+	uint64 reserved_memory = 3;
+	uint64 assigned_memory = 4;
+	bool slp_active = 5;
+	bool balancing_enabled = 6;
+	bool dm_operation_in_progress = 7;
+}

vendor/github.com/Microsoft/hcsshim/go.mod

@@ -3,27 +3,25 @@ module github.com/Microsoft/hcsshim
 go 1.12
 
 require (
-	github.com/Microsoft/go-winio v0.4.14
+	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
 	github.com/blang/semver v3.1.0+incompatible // indirect
+	github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f
 	github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1
 	github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69
 	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
 	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
 	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
-	github.com/containerd/ttrpc v0.0.0-20190826154248-f969a7f076a2
+	github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de
 	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
-	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
-	github.com/gogo/googleapis v1.2.0 // indirect
 	github.com/gogo/protobuf v1.2.1
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874 // indirect
 	github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f // indirect
-	github.com/opencontainers/runtime-spec v0.0.0-20190207185410-29686dbc5559
+	github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700
 	github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39
 	github.com/pkg/errors v0.8.1
+	github.com/prometheus/procfs v0.0.5 // indirect
 	github.com/sirupsen/logrus v1.4.1
 	github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 // indirect
 	github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5
@@ -32,7 +30,7 @@ require (
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f // indirect
 	go.opencensus.io v0.22.0
 	golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6
-	golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b
+	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
 	google.golang.org/grpc v1.20.1
 	gotest.tools v2.2.0+incompatible // indirect
 	k8s.io/kubernetes v1.13.0

vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go

@@ -16,6 +16,7 @@ const (
 	OutBoundNAT   EndpointPolicyType = "OutBoundNAT"
 	SDNRoute      EndpointPolicyType = "SDNRoute"
 	L4Proxy       EndpointPolicyType = "L4Proxy"
+	L4WFPPROXY    EndpointPolicyType = "L4WFPPROXY"
 	PortName      EndpointPolicyType = "PortName"
 	EncapOverhead EndpointPolicyType = "EncapOverhead"
 	// Endpoint and Network have InterfaceConstraint and ProviderAddress
@@ -128,6 +129,7 @@ type QosPolicySetting struct {
 type OutboundNatPolicySetting struct {
 	VirtualIP    string   `json:",omitempty"`
 	Exceptions   []string `json:",omitempty"`
+	Destinations []string `json:",omitempty"`
 }
 
 // SDNRoutePolicySetting sets SDN Route on an Endpoint.
@@ -137,16 +139,6 @@ type SDNRoutePolicySetting struct {
 	NeedEncap         bool   `json:",omitempty"`
 }
 
-// A ProxyType is a type of proxy used by the L4 proxy policy.
-type ProxyType int
-
-const (
-	// ProxyTypeVFP specifies a Virtual Filtering Protocol proxy.
-	ProxyTypeVFP ProxyType = iota
-	// ProxyTypeWFP specifies a Windows Filtering Platform proxy.
-	ProxyTypeWFP
-)
-
 // FiveTuple is nested in L4ProxyPolicySetting for WFP support.
 type FiveTuple struct {
 	Protocols       string `json:",omitempty"`
@@ -157,20 +149,11 @@ type FiveTuple struct {
 	Priority        uint16 `json:",omitempty"`
 }
 
-// L4ProxyPolicySetting sets Layer-4 Proxy on an endpoint.
+// L4WfpProxyPolicySetting sets Layer-4 Proxy on an endpoint.
-type L4ProxyPolicySetting struct {
+type L4WfpProxyPolicySetting struct {
-	IP            string   `json:",omitempty"`
 	Port        string    `json:",omitempty"`
-	Protocol      uint32   `json:",omitempty"` // EX: TCP = 6, UDP = 17
-	ExceptionList []string `json:",omitempty"`
-	Destination   string   `json:","`
-	OutboundNat   bool     `json:",omitempty"`
-
-	// For the WFP proxy
 	FilterTuple FiveTuple `json:",omitempty"`
-	ProxyType     ProxyType `json:",omitempty"`
 	UserSID     string    `json:",omitempty"`
-	CompartmentID uint32    `json:",omitempty"`
 }
 
 // PortnameEndpointPolicySetting sets the port name for an endpoint.

vendor/github.com/Microsoft/hcsshim/internal/cow/cow.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	"github.com/Microsoft/hcsshim/internal/schema1"
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
 )
 
 // Process is the interface for an OS process running in a container or utility VM.
@@ -63,8 +64,10 @@ type Container interface {
 	Close() error
 	// ID returns the container ID.
 	ID() string
-	// Properties returns the requested container properties.
+	// Properties returns the requested container properties targeting a V1 schema container.
 	Properties(ctx context.Context, types ...schema1.PropertyType) (*schema1.ContainerProperties, error)
+	// PropertiesV2 returns the requested container properties targeting a V2 schema container.
+	PropertiesV2(ctx context.Context, types ...hcsschema.PropertyType) (*hcsschema.Properties, error)
 	// Start starts a container.
 	Start(ctx context.Context) error
 	// Shutdown sends a shutdown request to the container (but does not wait for

vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go

@@ -15,6 +15,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/schema1"
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
 	"github.com/Microsoft/hcsshim/internal/timeout"
 	"github.com/Microsoft/hcsshim/internal/vmcompute"
 	"go.opencensus.io/trace"
@@ -345,6 +346,7 @@ func (computeSystem *System) ExitError() error {
 	}
 }
 
+// Properties returns the requested container properties targeting a V1 schema container.
 func (computeSystem *System) Properties(ctx context.Context, types ...schema1.PropertyType) (*schema1.ContainerProperties, error) {
 	computeSystem.handleLock.RLock()
 	defer computeSystem.handleLock.RUnlock()
@@ -373,6 +375,35 @@ func (computeSystem *System) Properties(ctx context.Context, types ...schema1.Pr
 	return properties, nil
 }
 
+// PropertiesV2 returns the requested container properties targeting a V2 schema container.
+func (computeSystem *System) PropertiesV2(ctx context.Context, types ...hcsschema.PropertyType) (*hcsschema.Properties, error) {
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+
+	operation := "hcsshim::System::PropertiesV2"
+
+	queryBytes, err := json.Marshal(hcsschema.PropertyQuery{PropertyTypes: types})
+	if err != nil {
+		return nil, makeSystemError(computeSystem, operation, "", err, nil)
+	}
+
+	propertiesJSON, resultJSON, err := vmcompute.HcsGetComputeSystemProperties(ctx, computeSystem.handle, string(queryBytes))
+	events := processHcsResult(ctx, resultJSON)
+	if err != nil {
+		return nil, makeSystemError(computeSystem, operation, "", err, events)
+	}
+
+	if propertiesJSON == "" {
+		return nil, ErrUnexpectedValue
+	}
+	properties := &hcsschema.Properties{}
+	if err := json.Unmarshal([]byte(propertiesJSON), properties); err != nil {
+		return nil, makeSystemError(computeSystem, operation, "", err, nil)
+	}
+
+	return properties, nil
+}
+
 // Pause pauses the execution of the computeSystem. This feature is not enabled in TP5.
 func (computeSystem *System) Pause(ctx context.Context) (err error) {
 	operation := "hcsshim::System::Pause"

vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go

@@ -57,6 +57,7 @@ type OutboundNatPolicy struct {
 	Policy
 	VIP          string   `json:"VIP,omitempty"`
 	Exceptions   []string `json:"ExceptionList,omitempty"`
+	Destinations []string `json:",omitempty"`
 }
 
 type ActionType string

vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go

@@ -22,4 +22,9 @@ type Memory2 struct {
 
 	// EnableDeferredCommit is private in the schema. If regenerated need to add back.
 	EnableDeferredCommit bool `json:"EnableDeferredCommit,omitempty"`
+
+	// EnableColdDiscardHint if enabled, then the memory cold discard hint feature is exposed
+	// to the VM, allowing it to trim non-zeroed pages from the working set (if supported by
+	// the guest operating system).
+	EnableColdDiscardHint bool `json:"EnableColdDiscardHint,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go

@@ -10,7 +10,7 @@
 package hcsschema
 
 type MemoryInformationForVm struct {
-	VirtualNodeCount int32 `json:"VirtualNodeCount,omitempty"`
+	VirtualNodeCount uint32 `json:"VirtualNodeCount,omitempty"`
 
 	VirtualMachineMemory *VmMemory `json:"VirtualMachineMemory,omitempty"`
 

vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go

@@ -11,9 +11,9 @@ package hcsschema
 
 //  Memory runtime statistics
 type MemoryStats struct {
-	MemoryUsageCommitBytes int32 `json:"MemoryUsageCommitBytes,omitempty"`
+	MemoryUsageCommitBytes uint64 `json:"MemoryUsageCommitBytes,omitempty"`
 
-	MemoryUsageCommitPeakBytes int32 `json:"MemoryUsageCommitPeakBytes,omitempty"`
+	MemoryUsageCommitPeakBytes uint64 `json:"MemoryUsageCommitPeakBytes,omitempty"`
 
-	MemoryUsagePrivateWorkingSetBytes int32 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"`
+	MemoryUsagePrivateWorkingSetBytes uint64 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go

@@ -11,9 +11,9 @@ package hcsschema
 
 //  CPU runtime statistics
 type ProcessorStats struct {
-	TotalRuntime100ns int32 `json:"TotalRuntime100ns,omitempty"`
+	TotalRuntime100ns uint64 `json:"TotalRuntime100ns,omitempty"`
 
-	RuntimeUser100ns int32 `json:"RuntimeUser100ns,omitempty"`
+	RuntimeUser100ns uint64 `json:"RuntimeUser100ns,omitempty"`
 
-	RuntimeKernel100ns int32 `json:"RuntimeKernel100ns,omitempty"`
+	RuntimeKernel100ns uint64 `json:"RuntimeKernel100ns,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go

@@ -9,6 +9,10 @@
 
 package hcsschema
 
+import (
+	v1 "github.com/containerd/cgroups/stats/v1"
+)
+
 type Properties struct {
 	Id string `json:"Id,omitempty"`
 
@@ -43,4 +47,8 @@ type Properties struct {
 	SharedMemoryRegionInfo []SharedMemoryRegionInfo `json:"SharedMemoryRegionInfo,omitempty"`
 
 	GuestConnectionInfo *GuestConnectionInfo `json:"GuestConnectionInfo,omitempty"`
+
+	// Metrics is not part of the API for HCS but this is used for LCOW v2 to
+	// return the full cgroup metrics from the guest.
+	Metrics *v1.Metrics `json:"LCOWMetrics,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go

@@ -11,5 +11,5 @@ package hcsschema
 
 //   By default the basic properties will be returned. This query provides a way to  request specific properties.
 type PropertyQuery struct {
-	PropertyTypes []string `json:"PropertyTypes,omitempty"`
+	PropertyTypes []PropertyType `json:"PropertyTypes,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/property_type.go

@@ -0,0 +1,23 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.1
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type PropertyType string
+
+const (
+	PTMemory                      PropertyType = "Memory"
+	PTGuestMemory                 PropertyType = "GuestMemory"
+	PTStatistics                  PropertyType = "Statistics"
+	PTProcessList                 PropertyType = "ProcessList"
+	PTTerminateOnLastHandleClosed PropertyType = "TerminateOnLastHandleClosed"
+	PTSharedMemoryRegion          PropertyType = "SharedMemoryRegion"
+	PTGuestConnection             PropertyType = "GuestConnection"
+	PTICHeartbeatStatus           PropertyType = "ICHeartbeatStatus"
+)

vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go

@@ -19,7 +19,7 @@ type Statistics struct {
 
 	ContainerStartTime time.Time `json:"ContainerStartTime,omitempty"`
 
-	Uptime100ns int32 `json:"Uptime100ns,omitempty"`
+	Uptime100ns uint64 `json:"Uptime100ns,omitempty"`
 
 	Processor *ProcessorStats `json:"Processor,omitempty"`
 

vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go

@@ -11,11 +11,11 @@ package hcsschema
 
 //  Storage runtime statistics
 type StorageStats struct {
-	ReadCountNormalized int32 `json:"ReadCountNormalized,omitempty"`
+	ReadCountNormalized uint64 `json:"ReadCountNormalized,omitempty"`
 
-	ReadSizeBytes int32 `json:"ReadSizeBytes,omitempty"`
+	ReadSizeBytes uint64 `json:"ReadSizeBytes,omitempty"`
 
-	WriteCountNormalized int32 `json:"WriteCountNormalized,omitempty"`
+	WriteCountNormalized uint64 `json:"WriteCountNormalized,omitempty"`
 
-	WriteSizeBytes int32 `json:"WriteSizeBytes,omitempty"`
+	WriteSizeBytes uint64 `json:"WriteSizeBytes,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go

@@ -14,9 +14,9 @@ type VmMemory struct {
 
 	AvailableMemoryBuffer int32 `json:"AvailableMemoryBuffer,omitempty"`
 
-	ReservedMemory int32 `json:"ReservedMemory,omitempty"`
+	ReservedMemory uint64 `json:"ReservedMemory,omitempty"`
 
-	AssignedMemory int32 `json:"AssignedMemory,omitempty"`
+	AssignedMemory uint64 `json:"AssignedMemory,omitempty"`
 
 	SlpActive bool `json:"SlpActive,omitempty"`
 

vendor/github.com/Microsoft/hcsshim/internal/wclayer/expandscratchsize.go

@@ -1,7 +1,13 @@
 package wclayer
 
 import (
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+
 	"github.com/Microsoft/hcsshim/internal/hcserror"
+	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/sirupsen/logrus"
 )
 
@@ -26,5 +32,114 @@ func ExpandScratchSize(path string, size uint64) (err error) {
 	if err != nil {
 		return hcserror.New(err, title+" - failed", "")
 	}
+
+	// Manually expand the volume now in order to work around bugs in 19H1 and
+	// prerelease versions of Vb. Remove once this is fixed in Windows.
+	if build := osversion.Get().Build; build >= osversion.V19H1 && build < 19020 {
+		err = expandSandboxVolume(path)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type virtualStorageType struct {
+	DeviceID uint32
+	VendorID [16]byte
+}
+
+type openVersion2 struct {
+	GetInfoOnly    int32    // bool but 4-byte aligned
+	ReadOnly       int32    // bool but 4-byte aligned
+	ResiliencyGUID [16]byte // GUID
+}
+
+type openVirtualDiskParameters struct {
+	Version  uint32 // Must always be set to 2
+	Version2 openVersion2
+}
+
+func attachVhd(path string) (syscall.Handle, error) {
+	var (
+		defaultType virtualStorageType
+		handle      syscall.Handle
+	)
+	parameters := openVirtualDiskParameters{Version: 2}
+	err := openVirtualDisk(
+		&defaultType,
+		path,
+		0,
+		0,
+		&parameters,
+		&handle)
+	if err != nil {
+		return 0, &os.PathError{Op: "OpenVirtualDisk", Path: path, Err: err}
+	}
+	err = attachVirtualDisk(handle, 0, 0, 0, 0, 0)
+	if err != nil {
+		syscall.Close(handle)
+		return 0, &os.PathError{Op: "AttachVirtualDisk", Path: path, Err: err}
+	}
+	return handle, nil
+}
+
+func expandSandboxVolume(path string) error {
+	// Mount the sandbox VHD temporarily.
+	vhdPath := filepath.Join(path, "sandbox.vhdx")
+	vhd, err := attachVhd(vhdPath)
+	if err != nil {
+		return &os.PathError{Op: "OpenVirtualDisk", Path: vhdPath, Err: err}
+	}
+	defer syscall.Close(vhd)
+
+	// Open the volume.
+	volumePath, err := GetLayerMountPath(path)
+	if err != nil {
+		return err
+	}
+	if volumePath[len(volumePath)-1] == '\\' {
+		volumePath = volumePath[:len(volumePath)-1]
+	}
+	volume, err := os.OpenFile(volumePath, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	defer volume.Close()
+
+	// Get the volume's underlying partition size in NTFS clusters.
+	var (
+		partitionSize int64
+		bytes         uint32
+	)
+	const _IOCTL_DISK_GET_LENGTH_INFO = 0x0007405C
+	err = syscall.DeviceIoControl(syscall.Handle(volume.Fd()), _IOCTL_DISK_GET_LENGTH_INFO, nil, 0, (*byte)(unsafe.Pointer(&partitionSize)), 8, &bytes, nil)
+	if err != nil {
+		return &os.PathError{Op: "IOCTL_DISK_GET_LENGTH_INFO", Path: volume.Name(), Err: err}
+	}
+	const (
+		clusterSize = 4096
+		sectorSize  = 512
+	)
+	targetClusters := partitionSize / clusterSize
+
+	// Get the volume's current size in NTFS clusters.
+	var volumeSize int64
+	err = getDiskFreeSpaceEx(volume.Name()+"\\", nil, &volumeSize, nil)
+	if err != nil {
+		return &os.PathError{Op: "GetDiskFreeSpaceEx", Path: volume.Name(), Err: err}
+	}
+	volumeClusters := volumeSize / clusterSize
+
+	// Only resize the volume if there is space to grow, otherwise this will
+	// fail with invalid parameter. NTFS reserves one cluster.
+	if volumeClusters+1 < targetClusters {
+		targetSectors := targetClusters * (clusterSize / sectorSize)
+		const _FSCTL_EXTEND_VOLUME = 0x000900F0
+		err = syscall.DeviceIoControl(syscall.Handle(volume.Fd()), _FSCTL_EXTEND_VOLUME, (*byte)(unsafe.Pointer(&targetSectors)), 8, nil, 0, &bytes, nil)
+		if err != nil {
+			return &os.PathError{Op: "FSCTL_EXTEND_VOLUME", Path: volume.Name(), Err: err}
+		}
+	}
 	return nil
 }

vendor/github.com/Microsoft/hcsshim/internal/wclayer/wclayer.go

@@ -24,4 +24,9 @@ import "github.com/Microsoft/go-winio/pkg/guid"
 
 //sys grantVmAccess(vmid string, filepath string) (hr error) = vmcompute.GrantVmAccess?
 
+//sys openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) [failretval != 0] = virtdisk.OpenVirtualDisk
+//sys attachVirtualDisk(handle syscall.Handle, sd uintptr, flags uint32, providerFlags uint32, params uintptr, overlapped uintptr) (err error) [failretval != 0] = virtdisk.AttachVirtualDisk
+
+//sys getDiskFreeSpaceEx(directoryName string, freeBytesAvailableToCaller *int64, totalNumberOfBytes *int64, totalNumberOfFreeBytes *int64) (err error) = GetDiskFreeSpaceExW
+
 type _guid = guid.GUID

vendor/github.com/Microsoft/hcsshim/internal/wclayer/zsyscall_windows.go

@@ -38,6 +38,8 @@ func errnoErr(e syscall.Errno) error {
 
 var (
 	modvmcompute = windows.NewLazySystemDLL("vmcompute.dll")
+	modvirtdisk  = windows.NewLazySystemDLL("virtdisk.dll")
+	modkernel32  = windows.NewLazySystemDLL("kernel32.dll")
 
 	procActivateLayer       = modvmcompute.NewProc("ActivateLayer")
 	procCopyLayer           = modvmcompute.NewProc("CopyLayer")
@@ -57,6 +59,9 @@ var (
 	procProcessBaseImage    = modvmcompute.NewProc("ProcessBaseImage")
 	procProcessUtilityImage = modvmcompute.NewProc("ProcessUtilityImage")
 	procGrantVmAccess       = modvmcompute.NewProc("GrantVmAccess")
+	procOpenVirtualDisk     = modvirtdisk.NewProc("OpenVirtualDisk")
+	procAttachVirtualDisk   = modvirtdisk.NewProc("AttachVirtualDisk")
+	procGetDiskFreeSpaceExW = modkernel32.NewProc("GetDiskFreeSpaceExW")
 )
 
 func activateLayer(info *driverInfo, id string) (hr error) {
@@ -508,3 +513,57 @@ func _grantVmAccess(vmid *uint16, filepath *uint16) (hr error) {
 	}
 	return
 }
+
+func openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	return _openVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, flags, parameters, handle)
+}
+
+func _openVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, flags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall6(procOpenVirtualDisk.Addr(), 6, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(flags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(handle)))
+	if r1 != 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func attachVirtualDisk(handle syscall.Handle, sd uintptr, flags uint32, providerFlags uint32, params uintptr, overlapped uintptr) (err error) {
+	r1, _, e1 := syscall.Syscall6(procAttachVirtualDisk.Addr(), 6, uintptr(handle), uintptr(sd), uintptr(flags), uintptr(providerFlags), uintptr(params), uintptr(overlapped))
+	if r1 != 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func getDiskFreeSpaceEx(directoryName string, freeBytesAvailableToCaller *int64, totalNumberOfBytes *int64, totalNumberOfFreeBytes *int64) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(directoryName)
+	if err != nil {
+		return
+	}
+	return _getDiskFreeSpaceEx(_p0, freeBytesAvailableToCaller, totalNumberOfBytes, totalNumberOfFreeBytes)
+}
+
+func _getDiskFreeSpaceEx(directoryName *uint16, freeBytesAvailableToCaller *int64, totalNumberOfBytes *int64, totalNumberOfFreeBytes *int64) (err error) {
+	r1, _, e1 := syscall.Syscall6(procGetDiskFreeSpaceExW.Addr(), 4, uintptr(unsafe.Pointer(directoryName)), uintptr(unsafe.Pointer(freeBytesAvailableToCaller)), uintptr(unsafe.Pointer(totalNumberOfBytes)), uintptr(unsafe.Pointer(totalNumberOfFreeBytes)), 0, 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}

vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go

@@ -0,0 +1,57 @@
+package osversion
+
+import (
+	"fmt"
+
+	"golang.org/x/sys/windows"
+)
+
+// OSVersion is a wrapper for Windows version information
+// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724439(v=vs.85).aspx
+type OSVersion struct {
+	Version      uint32
+	MajorVersion uint8
+	MinorVersion uint8
+	Build        uint16
+}
+
+// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724833(v=vs.85).aspx
+type osVersionInfoEx struct {
+	OSVersionInfoSize uint32
+	MajorVersion      uint32
+	MinorVersion      uint32
+	BuildNumber       uint32
+	PlatformID        uint32
+	CSDVersion        [128]uint16
+	ServicePackMajor  uint16
+	ServicePackMinor  uint16
+	SuiteMask         uint16
+	ProductType       byte
+	Reserve           byte
+}
+
+// Get gets the operating system version on Windows.
+// The calling application must be manifested to get the correct version information.
+func Get() OSVersion {
+	var err error
+	osv := OSVersion{}
+	osv.Version, err = windows.GetVersion()
+	if err != nil {
+		// GetVersion never fails.
+		panic(err)
+	}
+	osv.MajorVersion = uint8(osv.Version & 0xFF)
+	osv.MinorVersion = uint8(osv.Version >> 8 & 0xFF)
+	osv.Build = uint16(osv.Version >> 16)
+	return osv
+}
+
+// Build gets the build-number on Windows
+// The calling application must be manifested to get the correct version information.
+func Build() uint16 {
+	return Get().Build
+}
+
+func (osv OSVersion) ToString() string {
+	return fmt.Sprintf("%d.%d.%d", osv.MajorVersion, osv.MinorVersion, osv.Build)
+}

vendor/github.com/Microsoft/hcsshim/osversion/windowsbuilds.go

@@ -0,0 +1,27 @@
+package osversion
+
+const (
+	// RS1 (version 1607, codename "Redstone 1") corresponds to Windows Server
+	// 2016 (ltsc2016) and Windows 10 (Anniversary Update).
+	RS1 = 14393
+
+	// RS2 (version 1703, codename "Redstone 2") was a client-only update, and
+	// corresponds to Windows 10 (Creators Update).
+	RS2 = 15063
+
+	// RS3 (version 1709, codename "Redstone 3") corresponds to Windows Server
+	// 1709 (Semi-Annual Channel (SAC)), and Windows 10 (Fall Creators Update).
+	RS3 = 16299
+
+	// RS4 (version 1803, codename "Redstone 4") corresponds to Windows Server
+	// 1803 (Semi-Annual Channel (SAC)), and Windows 10 (April 2018 Update).
+	RS4 = 17134
+
+	// RS5 (version 1809, codename "Redstone 5") corresponds to Windows Server
+	// 2019 (ltsc2019), and Windows 10 (October 2018 Update).
+	RS5 = 17763
+
+	// V19H1 (version 1903) corresponds to Windows Sever 1903 (semi-annual
+	// channel).
+	V19H1 = 18362
+)