Dismount sandbox VHD on snapshot remove

Signed-off-by: Justin Terry (VM) <juterry@microsoft.com>

snapshots/windows/windows.go

@@ -27,6 +27,7 @@ import (
 	"syscall"
 	"unsafe"
 
+	"github.com/Microsoft/go-winio/vhd"
 	"github.com/Microsoft/hcsshim"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/log"
@@ -206,10 +207,21 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
 
 	path := s.getSnapshotDir(id)
 	renamedID := "rm-" + id
-	renamed := filepath.Join(s.root, "snapshots", renamedID)
+	renamed := s.getSnapshotDir(renamedID)
 	if err := os.Rename(path, renamed); err != nil && !os.IsNotExist(err) {
+		if !os.IsPermission(err) {
 			return err
 		}
+		// If permission denied, it's possible that the scratch is still mounted, an
+		// artifact after a hard daemon crash for example. Worth a shot to try detaching it
+		// before retrying the rename.
+		if detachErr := vhd.DetachVhd(filepath.Join(path, "sandbox.vhdx")); detachErr != nil {
+			return errors.Wrapf(err, "failed to detach VHD: %s", detachErr)
+		}
+		if renameErr := os.Rename(path, renamed); renameErr != nil && !os.IsNotExist(renameErr) {
+			return errors.Wrapf(err, "second rename attempt following detach failed: %s", renameErr)
+		}
+	}
 
 	if err := t.Commit(); err != nil {
 		if err1 := os.Rename(renamed, path); err1 != nil {

vendor.conf

@@ -32,7 +32,7 @@ github.com/opencontainers/image-spec v1.0.1
 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0
-github.com/Microsoft/go-winio v0.4.10
+github.com/Microsoft/go-winio v0.4.11
 github.com/Microsoft/hcsshim v0.7.9
 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4

vendor/github.com/Microsoft/go-winio/vhd/vhd.go

@@ -0,0 +1,108 @@
+// +build windows
+
+package vhd
+
+import "syscall"
+
+//go:generate go run mksyscall_windows.go -output zvhd.go vhd.go
+
+//sys createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.CreateVirtualDisk
+//sys openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *uintptr, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.OpenVirtualDisk
+//sys detachVirtualDisk(handle syscall.Handle, flags uint32, providerSpecificFlags uint32) (err error) [failretval != 0] = VirtDisk.DetachVirtualDisk
+
+type virtualStorageType struct {
+	DeviceID uint32
+	VendorID [16]byte
+}
+
+const virtualDiskAccessNONE uint32 = 0
+const virtualDiskAccessATTACHRO uint32 = 65536
+const virtualDiskAccessATTACHRW uint32 = 131072
+const virtualDiskAccessDETACH uint32 = 262144
+const virtualDiskAccessGETINFO uint32 = 524288
+const virtualDiskAccessCREATE uint32 = 1048576
+const virtualDiskAccessMETAOPS uint32 = 2097152
+const virtualDiskAccessREAD uint32 = 851968
+const virtualDiskAccessALL uint32 = 4128768
+const virtualDiskAccessWRITABLE uint32 = 3276800
+
+const createVirtualDiskFlagNone uint32 = 0
+const createVirtualDiskFlagFullPhysicalAllocation uint32 = 1
+const createVirtualDiskFlagPreventWritesToSourceDisk uint32 = 2
+const createVirtualDiskFlagDoNotCopyMetadataFromParent uint32 = 4
+
+type version2 struct {
+	UniqueID                 [16]byte // GUID
+	MaximumSize              uint64
+	BlockSizeInBytes         uint32
+	SectorSizeInBytes        uint32
+	ParentPath               *uint16 // string
+	SourcePath               *uint16 // string
+	OpenFlags                uint32
+	ParentVirtualStorageType virtualStorageType
+	SourceVirtualStorageType virtualStorageType
+	ResiliencyGUID           [16]byte // GUID
+}
+
+type createVirtualDiskParameters struct {
+	Version  uint32 // Must always be set to 2
+	Version2 version2
+}
+
+// CreateVhdx will create a simple vhdx file at the given path using default values.
+func CreateVhdx(path string, maxSizeInGb, blockSizeInMb uint32) error {
+	var defaultType virtualStorageType
+
+	parameters := createVirtualDiskParameters{
+		Version: 2,
+		Version2: version2{
+			MaximumSize:      uint64(maxSizeInGb) * 1024 * 1024 * 1024,
+			BlockSizeInBytes: blockSizeInMb * 1024 * 1024,
+		},
+	}
+
+	var handle syscall.Handle
+
+	if err := createVirtualDisk(
+		&defaultType,
+		path,
+		virtualDiskAccessNONE,
+		nil,
+		createVirtualDiskFlagNone,
+		0,
+		&parameters,
+		nil,
+		&handle); err != nil {
+		return err
+	}
+
+	if err := syscall.CloseHandle(handle); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// DetachVhd detaches a VHD attached at the given path.
+func DetachVhd(path string) error {
+	var (
+		defaultType virtualStorageType
+		handle      syscall.Handle
+	)
+
+	if err := openVirtualDisk(
+		&defaultType,
+		path,
+		virtualDiskAccessDETACH,
+		0,
+		nil,
+		&handle); err != nil {
+		return err
+	}
+	defer syscall.CloseHandle(handle)
+
+	if err := detachVirtualDisk(handle, 0, 0); err != nil {
+		return err
+	}
+	return nil
+}

vendor/github.com/Microsoft/go-winio/vhd/zvhd.go

@@ -0,0 +1,99 @@
+// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT
+
+package vhd
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var _ unsafe.Pointer
+
+// Do the interface allocations only once for common
+// Errno values.
+const (
+	errnoERROR_IO_PENDING = 997
+)
+
+var (
+	errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
+)
+
+// errnoErr returns common boxed Errno values, to prevent
+// allocations at runtime.
+func errnoErr(e syscall.Errno) error {
+	switch e {
+	case 0:
+		return nil
+	case errnoERROR_IO_PENDING:
+		return errERROR_IO_PENDING
+	}
+	// TODO: add more here, after collecting data on the common
+	// error values see on Windows. (perhaps when running
+	// all.bat?)
+	return e
+}
+
+var (
+	modVirtDisk = windows.NewLazySystemDLL("VirtDisk.dll")
+
+	procCreateVirtualDisk = modVirtDisk.NewProc("CreateVirtualDisk")
+	procOpenVirtualDisk   = modVirtDisk.NewProc("OpenVirtualDisk")
+	procDetachVirtualDisk = modVirtDisk.NewProc("DetachVirtualDisk")
+)
+
+func createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	return _createVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, securityDescriptor, flags, providerSpecificFlags, parameters, o, handle)
+}
+
+func _createVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall9(procCreateVirtualDisk.Addr(), 9, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(flags), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(handle)))
+	if r1 != 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *uintptr, handle *syscall.Handle) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	return _openVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, flags, parameters, handle)
+}
+
+func _openVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, flags uint32, parameters *uintptr, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall6(procOpenVirtualDisk.Addr(), 6, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(flags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(handle)))
+	if r1 != 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func detachVirtualDisk(handle syscall.Handle, flags uint32, providerSpecificFlags uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procDetachVirtualDisk.Addr(), 3, uintptr(handle), uintptr(flags), uintptr(providerSpecificFlags))
+	if r1 != 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}