Introduce the Windows lcow diff/snaphotter

Implements the Windows lcow differ/snapshotter responsible for managing
the creation and lifetime of lcow containers on Windows.

Signed-off-by: Justin Terry (VM) <juterry@microsoft.com>

cmd/containerd/builtins_windows.go

@@ -19,7 +19,9 @@
 package main
 
 import (
+	_ "github.com/containerd/containerd/diff/lcow"
 	_ "github.com/containerd/containerd/diff/windows"
+	_ "github.com/containerd/containerd/snapshots/lcow"
 	_ "github.com/containerd/containerd/snapshots/windows"
 	_ "github.com/containerd/containerd/windows"
 )

cmd/containerd/builtins_windows_v2.go

@@ -19,7 +19,9 @@
 package main
 
 import (
+	_ "github.com/containerd/containerd/diff/lcow"
 	_ "github.com/containerd/containerd/diff/windows"
 	_ "github.com/containerd/containerd/runtime/v2"
+	_ "github.com/containerd/containerd/snapshots/lcow"
 	_ "github.com/containerd/containerd/snapshots/windows"
 )

diff/lcow/lcow.go

@@ -0,0 +1,184 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package lcow
+
+import (
+	"context"
+	"io"
+	"os/exec"
+	"path"
+	"time"
+
+	"github.com/containerd/containerd/archive/compression"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/diff"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/metadata"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/plugin"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	plugin.Register(&plugin.Registration{
+		Type: plugin.DiffPlugin,
+		ID:   "windows-lcow",
+		Requires: []plugin.Type{
+			plugin.MetadataPlugin,
+		},
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			md, err := ic.Get(plugin.MetadataPlugin)
+			if err != nil {
+				return nil, err
+			}
+
+			ic.Meta.Platforms = append(ic.Meta.Platforms, ocispec.Platform{
+				OS:           "linux",
+				Architecture: "amd64",
+			})
+			return NewWindowsLcowDiff(md.(*metadata.DB).ContentStore())
+		},
+	})
+}
+
+// CompareApplier handles both comparison and
+// application of layer diffs.
+type CompareApplier interface {
+	diff.Applier
+	diff.Comparer
+}
+
+// windowsLcowDiff does filesystem comparison and application
+// for Windows specific Linux layer diffs.
+type windowsLcowDiff struct {
+	store content.Store
+}
+
+var emptyDesc = ocispec.Descriptor{}
+
+// NewWindowsLcowDiff is the Windows LCOW container layer implementation
+// for comparing and applying Linux filesystem layers on Windows
+func NewWindowsLcowDiff(store content.Store) (CompareApplier, error) {
+	return windowsLcowDiff{
+		store: store,
+	}, nil
+}
+
+// Apply applies the content associated with the provided digests onto the
+// provided mounts. Archive content will be extracted and decompressed if
+// necessary.
+func (s windowsLcowDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount) (d ocispec.Descriptor, err error) {
+	t1 := time.Now()
+	defer func() {
+		if err == nil {
+			log.G(ctx).WithFields(logrus.Fields{
+				"d":     time.Now().Sub(t1),
+				"dgst":  desc.Digest,
+				"size":  desc.Size,
+				"media": desc.MediaType,
+			}).Debugf("diff applied")
+		}
+	}()
+
+	layer, _, err := mountsToLayerAndParents(mounts)
+	if err != nil {
+		return emptyDesc, err
+	}
+
+	isCompressed, err := images.IsCompressedDiff(ctx, desc.MediaType)
+	if err != nil {
+		return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType)
+	}
+
+	ra, err := s.store.ReaderAt(ctx, desc)
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to get reader from content store")
+	}
+	defer ra.Close()
+	rdr := content.NewReader(ra)
+	if isCompressed {
+		ds, err := compression.DecompressStream(rdr)
+		if err != nil {
+			return emptyDesc, err
+		}
+		defer ds.Close()
+		rdr = ds
+	}
+	// Calculate the Digest as we go
+	digester := digest.Canonical.Digester()
+	rc := &readCounter{
+		r: io.TeeReader(rdr, digester.Hash()),
+	}
+
+	cmd := exec.Command(
+		"runhcs.exe",
+		"tar2vhd",
+		"--scratchpath", path.Join(layer, "sandbox.vhdx"), // TODO: JTERRY75 when the snapshotter changes this to be scratch.vhdx update it here too.
+		"--destpath", path.Join(layer, "layer.vhd"))
+
+	cmd.Stdin = rc
+	if bytes, err := cmd.CombinedOutput(); err != nil {
+		return emptyDesc, errors.Wrapf(err, "failed to exec runhcs.exe tar2vhd: %s", string(bytes))
+	}
+
+	return ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageLayer,
+		Size:      rc.c,
+		Digest:    digester.Digest(),
+	}, nil
+}
+
+// Compare creates a diff between the given mounts and uploads the result
+// to the content store.
+func (s windowsLcowDiff) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) {
+	return emptyDesc, errdefs.ErrNotImplemented
+}
+
+type readCounter struct {
+	r io.Reader
+	c int64
+}
+
+func (rc *readCounter) Read(p []byte) (n int, err error) {
+	n, err = rc.r.Read(p)
+	rc.c += int64(n)
+	return
+}
+
+func mountsToLayerAndParents(mounts []mount.Mount) (string, []string, error) {
+	if len(mounts) != 1 {
+		return "", nil, errors.Wrap(errdefs.ErrInvalidArgument, "number of mounts should always be 1 for Windows lcow-layers")
+	}
+	mnt := mounts[0]
+	if mnt.Type != "lcow-layer" {
+		return "", nil, errors.Wrap(errdefs.ErrInvalidArgument, "mount layer type must be lcow-layer")
+	}
+
+	parentLayerPaths, err := mnt.GetParentPaths()
+	if err != nil {
+		return "", nil, err
+	}
+
+	return mnt.Source, parentLayerPaths, nil
+}

diff/windows/windows.go

@@ -175,12 +175,18 @@ func mountsToLayerAndParents(mounts []mount.Mount) (string, []string, error) {
 	if len(mounts) != 1 {
 		return "", nil, errors.Wrap(errdefs.ErrInvalidArgument, "number of mounts should always be 1 for Windows layers")
 	}
-	layer := mounts[0].Source
+	mnt := mounts[0]
+	if mnt.Type != "windows-layer" {
+		// This is a special case error. When this is received the diff service
+		// will attempt the next differ in the chain which for Windows is the
+		// lcow differ that we want.
+		return "", nil, errdefs.ErrNotImplemented
+	}
 
-	parentLayerPaths, err := mounts[0].GetParentPaths()
+	parentLayerPaths, err := mnt.GetParentPaths()
 	if err != nil {
 		return "", nil, err
 	}
 
-	return layer, parentLayerPaths, nil
+	return mnt.Source, parentLayerPaths, nil
 }

mount/mount_windows.go

@@ -32,6 +32,10 @@ var (
 
 // Mount to the provided target
 func (m *Mount) Mount(target string) error {
+	if m.Type != "windows-layer" {
+		return errors.Errorf("invalid windows mount type: '%s'", m.Type)
+	}
+
 	home, layerID := filepath.Split(m.Source)
 
 	parentLayerPaths, err := m.GetParentPaths()

platforms/defaults.go

@@ -22,11 +22,6 @@ import (
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// Default returns the default matcher for the platform.
-func Default() MatchComparer {
-	return Only(DefaultSpec())
-}
-
 // DefaultString returns the default string specifier for the platform.
 func DefaultString() string {
 	return Format(DefaultSpec())

platforms/defaults_unix.go

@@ -0,0 +1,24 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package platforms
+
+// Default returns the default matcher for the platform.
+func Default() MatchComparer {
+	return Only(DefaultSpec())
+}

platforms/defaults_windows.go

@@ -0,0 +1,31 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package platforms
+
+import (
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Default returns the default matcher for the platform.
+func Default() MatchComparer {
+	return Ordered(DefaultSpec(), specs.Platform{
+		OS:           "linux",
+		Architecture: "amd64",
+	})
+}

runtime/v2/manager.go

@@ -30,10 +30,8 @@ import (
 	"github.com/containerd/containerd/metadata"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/namespaces"
-	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/runtime"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 func init() {
@@ -44,7 +42,7 @@ func init() {
 			plugin.MetadataPlugin,
 		},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
-			ic.Meta.Platforms = []ocispec.Platform{platforms.DefaultSpec()}
+			ic.Meta.Platforms = supportedPlatforms()
 			if err := os.MkdirAll(ic.Root, 0711); err != nil {
 				return nil, err
 			}

runtime/v2/manager_unix.go

@@ -0,0 +1,28 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v2
+
+import (
+	"github.com/containerd/containerd/platforms"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func supportedPlatforms() []ocispec.Platform {
+	return []ocispec.Platform{platforms.DefaultSpec()}
+}

runtime/v2/manager_windows.go

@@ -0,0 +1,34 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v2
+
+import (
+	"github.com/containerd/containerd/platforms"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func supportedPlatforms() []ocispec.Platform {
+	return []ocispec.Platform{
+		platforms.DefaultSpec(),
+		{
+			OS:           "linux",
+			Architecture: "amd64",
+		},
+	}
+}

services/diff/service_windows.go

@@ -19,5 +19,5 @@
 package diff
 
 var defaultDifferConfig = &config{
-	Order: []string{"windows"},
+	Order: []string{"windows", "windows-lcow"},
 }

snapshots/lcow/lcow.go

@@ -0,0 +1,430 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package lcow
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/snapshots"
+	"github.com/containerd/containerd/snapshots/storage"
+	"github.com/containerd/continuity/fs"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/windows"
+)
+
+func init() {
+	plugin.Register(&plugin.Registration{
+		Type: plugin.SnapshotPlugin,
+		ID:   "windows-lcow",
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			ic.Meta.Platforms = append(ic.Meta.Platforms, ocispec.Platform{
+				OS:           "linux",
+				Architecture: "amd64",
+			})
+			return NewSnapshotter(ic.Root)
+		},
+	})
+}
+
+type snapshotter struct {
+	root string
+	ms   *storage.MetaStore
+}
+
+// NewSnapshotter returns a new windows snapshotter
+func NewSnapshotter(root string) (snapshots.Snapshotter, error) {
+	fsType, err := getFileSystemType(root)
+	if err != nil {
+		return nil, err
+	}
+	if strings.ToLower(fsType) != "ntfs" {
+		return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "%s is not on an NTFS volume - only NTFS volumes are supported", root)
+	}
+
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+	ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db"))
+	if err != nil {
+		return nil, err
+	}
+
+	if err := os.Mkdir(filepath.Join(root, "snapshots"), 0700); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+
+	return &snapshotter{
+		root: root,
+		ms:   ms,
+	}, nil
+}
+
+// Stat returns the info for an active or committed snapshot by name or
+// key.
+//
+// Should be used for parent resolution, existence checks and to discern
+// the kind of snapshot.
+func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) {
+	log.G(ctx).Debug("Starting Stat")
+	ctx, t, err := s.ms.TransactionContext(ctx, false)
+	if err != nil {
+		return snapshots.Info{}, err
+	}
+	defer t.Rollback()
+
+	_, info, _, err := storage.GetInfo(ctx, key)
+	return info, err
+}
+
+func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) {
+	log.G(ctx).Debug("Starting Update")
+	ctx, t, err := s.ms.TransactionContext(ctx, true)
+	if err != nil {
+		return snapshots.Info{}, err
+	}
+	defer t.Rollback()
+
+	info, err = storage.UpdateInfo(ctx, info, fieldpaths...)
+	if err != nil {
+		return snapshots.Info{}, err
+	}
+
+	if err := t.Commit(); err != nil {
+		return snapshots.Info{}, err
+	}
+
+	return info, nil
+}
+
+func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) {
+	log.G(ctx).Debug("Starting Usage")
+	ctx, t, err := s.ms.TransactionContext(ctx, false)
+	if err != nil {
+		return snapshots.Usage{}, err
+	}
+	defer t.Rollback()
+
+	_, info, usage, err := storage.GetInfo(ctx, key)
+	if err != nil {
+		return snapshots.Usage{}, err
+	}
+
+	if info.Kind == snapshots.KindActive {
+		du := fs.Usage{
+			Size: 0,
+		}
+		usage = snapshots.Usage(du)
+	}
+
+	return usage, nil
+}
+
+func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
+	log.G(ctx).Debug("Starting Prepare")
+	return s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts)
+}
+
+func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
+	log.G(ctx).Debug("Starting View")
+	return s.createSnapshot(ctx, snapshots.KindView, key, parent, opts)
+}
+
+// Mounts returns the mounts for the transaction identified by key. Can be
+// called on an read-write or readonly transaction.
+//
+// This can be used to recover mounts after calling View or Prepare.
+func (s *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) {
+	log.G(ctx).Debug("Starting Mounts")
+	ctx, t, err := s.ms.TransactionContext(ctx, false)
+	if err != nil {
+		return nil, err
+	}
+	defer t.Rollback()
+
+	snapshot, err := storage.GetSnapshot(ctx, key)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get snapshot mount")
+	}
+	return s.mounts(snapshot), nil
+}
+
+func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error {
+	log.G(ctx).Debug("Starting Commit")
+	ctx, t, err := s.ms.TransactionContext(ctx, true)
+	if err != nil {
+		return err
+	}
+	defer t.Rollback()
+
+	usage := fs.Usage{
+		Size: 0,
+	}
+
+	if _, err = storage.CommitActive(ctx, key, name, snapshots.Usage(usage), opts...); err != nil {
+		return errors.Wrap(err, "failed to commit snapshot")
+	}
+
+	if err := t.Commit(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Remove abandons the transaction identified by key. All resources
+// associated with the key will be removed.
+func (s *snapshotter) Remove(ctx context.Context, key string) error {
+	log.G(ctx).Debug("Starting Remove")
+	ctx, t, err := s.ms.TransactionContext(ctx, true)
+	if err != nil {
+		return err
+	}
+	defer t.Rollback()
+
+	id, _, err := storage.Remove(ctx, key)
+	if err != nil {
+		return errors.Wrap(err, "failed to remove")
+	}
+
+	path := s.getSnapshotDir(id)
+	renamed := s.getSnapshotDir("rm-" + id)
+	if err := os.Rename(path, renamed); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if err := t.Commit(); err != nil {
+		if err1 := os.Rename(renamed, path); err1 != nil {
+			// May cause inconsistent data on disk
+			log.G(ctx).WithError(err1).WithField("path", renamed).Errorf("Failed to rename after failed commit")
+		}
+		return errors.Wrap(err, "failed to commit")
+	}
+
+	if err := os.RemoveAll(renamed); err != nil {
+		// Must be cleaned up, any "rm-*" could be removed if no active transactions
+		log.G(ctx).WithError(err).WithField("path", renamed).Warnf("Failed to remove root filesystem")
+	}
+
+	return nil
+}
+
+// Walk the committed snapshots.
+func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
+	log.G(ctx).Debug("Starting Walk")
+	ctx, t, err := s.ms.TransactionContext(ctx, false)
+	if err != nil {
+		return err
+	}
+	defer t.Rollback()
+
+	return storage.WalkInfo(ctx, fn)
+}
+
+// Close closes the snapshotter
+func (s *snapshotter) Close() error {
+	return s.ms.Close()
+}
+
+func (s *snapshotter) mounts(sn storage.Snapshot) []mount.Mount {
+	var (
+		roFlag           string
+		source           string
+		parentLayerPaths []string
+	)
+
+	if sn.Kind == snapshots.KindView {
+		roFlag = "ro"
+	} else {
+		roFlag = "rw"
+	}
+
+	if len(sn.ParentIDs) == 0 || sn.Kind == snapshots.KindActive {
+		source = s.getSnapshotDir(sn.ID)
+		parentLayerPaths = s.parentIDsToParentPaths(sn.ParentIDs)
+	} else {
+		source = s.getSnapshotDir(sn.ParentIDs[0])
+		parentLayerPaths = s.parentIDsToParentPaths(sn.ParentIDs[1:])
+	}
+
+	// error is not checked here, as a string array will never fail to Marshal
+	parentLayersJSON, _ := json.Marshal(parentLayerPaths)
+	parentLayersOption := mount.ParentLayerPathsFlag + string(parentLayersJSON)
+
+	var mounts []mount.Mount
+	mounts = append(mounts, mount.Mount{
+		Source: source,
+		Type:   "lcow-layer",
+		Options: []string{
+			roFlag,
+			parentLayersOption,
+		},
+	})
+
+	return mounts
+}
+
+func (s *snapshotter) getSnapshotDir(id string) string {
+	return filepath.Join(s.root, "snapshots", id)
+}
+
+func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) ([]mount.Mount, error) {
+	ctx, t, err := s.ms.TransactionContext(ctx, true)
+	if err != nil {
+		return nil, err
+	}
+	defer t.Rollback()
+
+	newSnapshot, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create snapshot")
+	}
+
+	if kind == snapshots.KindActive {
+		log.G(ctx).Debug("createSnapshot active")
+		// Create the new snapshot dir
+		snDir := s.getSnapshotDir(newSnapshot.ID)
+		if err := os.MkdirAll(snDir, 0700); err != nil {
+			return nil, err
+		}
+		// Create the scratch.vhdx cache file if it doesnt already exit.
+		scratchPath := filepath.Join(s.root, "scratch.vhdx")
+		scratchLockPath := filepath.Join(s.root, "scratch.vhdx.lock")
+		startTime := time.Now()
+		timeout := 2 * time.Minute
+		var scratchSource *os.File
+		for {
+			var err error
+			scratchSource, err = os.OpenFile(scratchPath, os.O_RDONLY, 0700)
+			if err != nil {
+				if os.IsNotExist(err) {
+					// No scratch path. Take the lock and create it.
+					slock, err := os.OpenFile(scratchLockPath, os.O_EXCL|os.O_CREATE, 0700)
+					if err != nil {
+						if time.Now().Sub(startTime) >= timeout {
+							return nil, errors.Wrap(err, "timed out waiting for scratch.vhdx.lock")
+						}
+						// Couldnt obtain the lock. Sleep and try again.
+						time.Sleep(1 * time.Second)
+						continue
+					}
+					defer slock.Close()
+					// Create the scratch
+					cmd := exec.Command(
+						"runhcs.exe",
+						"create-scratch",
+						"--destpath", scratchPath)
+
+					if bytes, err := cmd.CombinedOutput(); err != nil {
+						_ = os.Remove(scratchPath)
+						return nil, errors.Wrapf(err, "failed to create scratch.vhdx. additional info: '%s'", string(bytes))
+					}
+					// Successfully created scratch in the cache. Open and copy
+					continue
+				} else {
+					if time.Now().Sub(startTime) >= timeout {
+						return nil, errors.Wrap(err, "timed out waiting for scratch.vhdx")
+					}
+					// Couldnt obtain read access. Sleep and try again. Likely
+					// this case is that scratch.vhdx is in the process of being
+					// written actively.
+					time.Sleep(1 * time.Second)
+					continue
+				}
+			}
+			break
+		}
+		defer scratchSource.Close()
+
+		// TODO: JTERRY75 - This has to be called sandbox.vhdx for the time
+		// being but it really is the scratch.vhdx Using this naming convention
+		// for now but this is not the kubernetes sandbox.
+		//
+		// Create the sandbox.vhdx for this snapshot from the cache.
+		destPath := filepath.Join(snDir, "sandbox.vhdx")
+		dest, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE, 0700)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create sandbox.vhdx in snapshot")
+		}
+		defer dest.Close()
+		if _, err := io.Copy(dest, scratchSource); err != nil {
+			dest.Close()
+			os.Remove(destPath)
+			return nil, errors.Wrap(err, "failed to copy cached scratch.vhdx to sandbox.vhdx in snapshot")
+		}
+	}
+
+	if err := t.Commit(); err != nil {
+		return nil, errors.Wrap(err, "commit failed")
+	}
+
+	return s.mounts(newSnapshot), nil
+}
+
+func (s *snapshotter) parentIDsToParentPaths(parentIDs []string) []string {
+	var parentLayerPaths []string
+	for _, ID := range parentIDs {
+		parentLayerPaths = append(parentLayerPaths, s.getSnapshotDir(ID))
+	}
+	return parentLayerPaths
+}
+
+// getFileSystemType obtains the type of a file system through GetVolumeInformation
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364993(v=vs.85).aspx
+func getFileSystemType(path string) (fsType string, hr error) {
+	drive := filepath.VolumeName(path)
+	if len(drive) != 2 {
+		return "", errors.New("getFileSystemType path must start with a drive letter")
+	}
+
+	var (
+		modkernel32              = windows.NewLazySystemDLL("kernel32.dll")
+		procGetVolumeInformation = modkernel32.NewProc("GetVolumeInformationW")
+		buf                      = make([]uint16, 255)
+		size                     = windows.MAX_PATH + 1
+	)
+	drive += `\`
+	n := uintptr(unsafe.Pointer(nil))
+	r0, _, _ := syscall.Syscall9(procGetVolumeInformation.Addr(), 8, uintptr(unsafe.Pointer(windows.StringToUTF16Ptr(drive))), n, n, n, n, n, uintptr(unsafe.Pointer(&buf[0])), uintptr(size), 0)
+	if int32(r0) < 0 {
+		hr = syscall.Errno(win32FromHresult(r0))
+	}
+	fsType = windows.UTF16ToString(buf)
+	return
+}
+
+// win32FromHresult is a helper function to get the win32 error code from an HRESULT
+func win32FromHresult(hr uintptr) uintptr {
+	if hr&0x1fff0000 == 0x00070000 {
+		return hr & 0xffff
+	}
+	return hr
+}

snapshots/windows/windows.go

@@ -206,7 +206,7 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
 
 	path := s.getSnapshotDir(id)
 	renamedID := "rm-" + id
-	renamed := filepath.Join(s.root, "snapshots", "rm-"+id)
+	renamed := filepath.Join(s.root, "snapshots", renamedID)
 	if err := os.Rename(path, renamed); err != nil && !os.IsNotExist(err) {
 		return err
 	}