go.mod: Bump hcsshim to v0.10.0-rc.1

This contains quite a bit (also bumps google/uuid to 1.3.0). Some HostProcess
container improvements to get ready for whenever it goes to stable in
Kubernetes, Hyper-V (windows) container support for CRI, and a plethora of
other small additions and fixes.

Signed-off-by: Daniel Canter <dcanter@microsoft.com>

vendor/github.com/Microsoft/hcsshim/ext4/dmverity/dmverity.go

@@ -1,6 +1,7 @@
 package dmverity
 
 import (
+	"bufio"
 	"bytes"
 	"crypto/rand"
 	"crypto/sha256"
@@ -12,19 +13,29 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/Microsoft/hcsshim/ext4/internal/compactext4"
+	"github.com/Microsoft/hcsshim/internal/memory"
 )
 
 const (
 	blockSize = compactext4.BlockSize
-	// RecommendedVHDSizeGB is the recommended size in GB for VHDs, which is not a hard limit. 
-	RecommendedVHDSizeGB = 128 * 1024 * 1024 * 1024
+	// MerkleTreeBufioSize is a default buffer size to use with bufio.Reader
+	MerkleTreeBufioSize = memory.MiB // 1MB
+	// RecommendedVHDSizeGB is the recommended size in GB for VHDs, which is not a hard limit.
+	RecommendedVHDSizeGB = 128 * memory.GiB
+	// VeritySignature is a value written to dm-verity super-block.
+	VeritySignature = "verity"
+)
+
+var (
+	salt   = bytes.Repeat([]byte{0}, 32)
+	sbSize = binary.Size(dmveritySuperblock{})
 )
-var salt = bytes.Repeat([]byte{0}, 32)
 
 var (
 	ErrSuperBlockReadFailure  = errors.New("failed to read dm-verity super block")
 	ErrSuperBlockParseFailure = errors.New("failed to parse dm-verity super block")
 	ErrRootHashReadFailure    = errors.New("failed to read dm-verity root hash")
+	ErrNotVeritySuperBlock    = errors.New("invalid dm-verity super-block signature")
 )
 
 type dmveritySuperblock struct {
@@ -69,20 +80,19 @@ type VerityInfo struct {
 	Version       uint32
 }
 
-// MerkleTree constructs dm-verity hash-tree for a given byte array with a fixed salt (0-byte) and algorithm (sha256).
-func MerkleTree(data []byte) ([]byte, error) {
+// MerkleTree constructs dm-verity hash-tree for a given io.Reader with a fixed salt (0-byte) and algorithm (sha256).
+func MerkleTree(r io.Reader) ([]byte, error) {
 	layers := make([][]byte, 0)
+	currentLevel := r
 
-	currentLevel := bytes.NewBuffer(data)
-
-	for currentLevel.Len() != blockSize {
-		blocks := currentLevel.Len() / blockSize
+	for {
 		nextLevel := bytes.NewBuffer(make([]byte, 0))
-
-		for i := 0; i < blocks; i++ {
+		for {
 			block := make([]byte, blockSize)
-			_, err := currentLevel.Read(block)
-			if err != nil {
+			if _, err := io.ReadFull(currentLevel, block); err != nil {
+				if err == io.EOF {
+					break
+				}
 				return nil, errors.Wrap(err, "failed to read data block")
 			}
 			h := hash2(salt, block)
@@ -92,14 +102,18 @@ func MerkleTree(data []byte) ([]byte, error) {
 		padding := bytes.Repeat([]byte{0}, blockSize-(nextLevel.Len()%blockSize))
 		nextLevel.Write(padding)
 
-		currentLevel = nextLevel
-		layers = append(layers, currentLevel.Bytes())
+		layers = append(layers, nextLevel.Bytes())
+		currentLevel = bufio.NewReaderSize(nextLevel, MerkleTreeBufioSize)
+
+		// This means that only root hash remains and our job is done
+		if nextLevel.Len() == blockSize {
+			break
+		}
 	}
 
-	var tree = bytes.NewBuffer(make([]byte, 0))
+	tree := bytes.NewBuffer(make([]byte, 0))
 	for i := len(layers) - 1; i >= 0; i-- {
-		_, err := tree.Write(layers[i])
-		if err != nil {
+		if _, err := tree.Write(layers[i]); err != nil {
 			return nil, errors.Wrap(err, "failed to write merkle tree")
 		}
 	}
@@ -125,7 +139,7 @@ func NewDMVeritySuperblock(size uint64) *dmveritySuperblock {
 		SaltSize:      uint16(len(salt)),
 	}
 
-	copy(superblock.Signature[:], "verity")
+	copy(superblock.Signature[:], VeritySignature)
 	copy(superblock.Algorithm[:], "sha256")
 	copy(superblock.Salt[:], salt)
 
@@ -165,7 +179,7 @@ func ReadDMVerityInfo(vhdPath string, offsetInBytes int64) (*VerityInfo, error)
 	block := make([]byte, blockSize)
 	if s, err := vhd.Read(block); err != nil || s != blockSize {
 		if err != nil {
-			return nil, errors.Wrapf(ErrSuperBlockReadFailure, "%s", err)
+			return nil, errors.Wrapf(err, "%s", ErrSuperBlockReadFailure)
 		}
 		return nil, errors.Wrapf(ErrSuperBlockReadFailure, "unexpected bytes read: expected=%d, actual=%d", blockSize, s)
 	}
@@ -173,13 +187,15 @@ func ReadDMVerityInfo(vhdPath string, offsetInBytes int64) (*VerityInfo, error)
 	dmvSB := &dmveritySuperblock{}
 	b := bytes.NewBuffer(block)
 	if err := binary.Read(b, binary.LittleEndian, dmvSB); err != nil {
-		return nil, errors.Wrapf(ErrSuperBlockParseFailure, "%s", err)
+		return nil, errors.Wrapf(err, "%s", ErrSuperBlockParseFailure)
+	}
+	if string(bytes.Trim(dmvSB.Signature[:], "\x00")[:]) != VeritySignature {
+		return nil, ErrNotVeritySuperBlock
 	}
-
 	// read the merkle tree root
 	if s, err := vhd.Read(block); err != nil || s != blockSize {
 		if err != nil {
-			return nil, errors.Wrapf(ErrRootHashReadFailure, "%s", err)
+			return nil, errors.Wrapf(err, "%s", ErrRootHashReadFailure)
 		}
 		return nil, errors.Wrapf(ErrRootHashReadFailure, "unexpected bytes read: expected=%d, actual=%d", blockSize, s)
 	}
@@ -196,3 +212,37 @@ func ReadDMVerityInfo(vhdPath string, offsetInBytes int64) (*VerityInfo, error)
 		Version:            dmvSB.Version,
 	}, nil
 }
+
+// ComputeAndWriteHashDevice builds merkle tree from a given io.ReadSeeker and writes the result
+// hash device (dm-verity super-block combined with merkle tree) to io.WriteSeeker.
+func ComputeAndWriteHashDevice(r io.ReadSeeker, w io.WriteSeeker) error {
+	if _, err := r.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+	tree, err := MerkleTree(r)
+	if err != nil {
+		return errors.Wrap(err, "failed to build merkle tree")
+	}
+
+	devSize, err := r.Seek(0, io.SeekEnd)
+	if err != nil {
+		return err
+	}
+	dmVeritySB := NewDMVeritySuperblock(uint64(devSize))
+	if _, err := w.Seek(0, io.SeekEnd); err != nil {
+		return err
+	}
+	if err := binary.Write(w, binary.LittleEndian, dmVeritySB); err != nil {
+		return errors.Wrap(err, "failed to write dm-verity super-block")
+	}
+	// write super-block padding
+	padding := bytes.Repeat([]byte{0}, blockSize-(sbSize%blockSize))
+	if _, err = w.Write(padding); err != nil {
+		return err
+	}
+	// write tree
+	if _, err := w.Write(tree); err != nil {
+		return errors.Wrap(err, "failed to write merkle tree")
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/ext4/internal/compactext4/compact.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/Microsoft/hcsshim/ext4/internal/format"
+	"github.com/Microsoft/hcsshim/internal/memory"
 )
 
 // Writer writes a compact ext4 file system.
@@ -101,15 +102,15 @@ const (
 	maxInodesPerGroup       = BlockSize * 8 // Limited by the inode bitmap
 	inodesPerGroupIncrement = BlockSize / inodeSize
 
-	defaultMaxDiskSize = 16 * 1024 * 1024 * 1024        // 16GB
+	defaultMaxDiskSize = 16 * memory.GiB                // 16GB
 	maxMaxDiskSize     = 16 * 1024 * 1024 * 1024 * 1024 // 16TB
 
 	groupDescriptorSize      = 32 // Use the small group descriptor
 	groupsPerDescriptorBlock = BlockSize / groupDescriptorSize
 
-	maxFileSize             = 128 * 1024 * 1024 * 1024 // 128GB file size maximum for now
-	smallSymlinkSize        = 59                       // max symlink size that goes directly in the inode
-	maxBlocksPerExtent      = 0x8000                   // maximum number of blocks in an extent
+	maxFileSize             = 128 * memory.GiB // 128GB file size maximum for now
+	smallSymlinkSize        = 59               // max symlink size that goes directly in the inode
+	maxBlocksPerExtent      = 0x8000           // maximum number of blocks in an extent
 	inodeDataSize           = 60
 	inodeUsedSize           = 152 // fields through CrtimeExtra
 	inodeExtraSize          = inodeSize - inodeUsedSize
@@ -414,6 +415,15 @@ func (w *Writer) makeInode(f *File, node *inode) (*inode, error) {
 	node.Devmajor = f.Devmajor
 	node.Devminor = f.Devminor
 	node.Data = nil
+	if f.Xattrs == nil {
+		f.Xattrs = make(map[string][]byte)
+	}
+
+	// copy over existing xattrs first, we need to merge existing xattrs and the passed xattrs.
+	existingXattrs := make(map[string][]byte)
+	if len(node.XattrInline) > 0 {
+		getXattrs(node.XattrInline[4:], existingXattrs, 0)
+	}
 	node.XattrInline = nil
 
 	var xstate xattrState
@@ -452,6 +462,13 @@ func (w *Writer) makeInode(f *File, node *inode) (*inode, error) {
 		return nil, fmt.Errorf("invalid mode %o", mode)
 	}
 
+	// merge xattrs but prefer currently passed over existing
+	for name, data := range existingXattrs {
+		if _, ok := f.Xattrs[name]; !ok {
+			f.Xattrs[name] = data
+		}
+	}
+
 	// Accumulate the extended attributes.
 	if len(f.Xattrs) != 0 {
 		// Sort the xattrs to avoid non-determinism in map iteration.
@@ -514,15 +531,16 @@ func (w *Writer) lookup(name string, mustExist bool) (*inode, *inode, string, er
 	return dir, child, childname, nil
 }
 
-// CreateWithParents adds a file to the file system creating the parent directories in the path if
-// they don't exist (like `mkdir -p`). These non existing parent directories are created
+// MakeParents ensures that all the parent directories in the path specified by `name` exists. If
+// they don't exist it creates them (like `mkdir -p`). These non existing parent directories are created
 // with the same permissions as that of it's parent directory. It is expected that the a
 // call to make these parent directories will be made at a later point with the correct
 // permissions, at that time the permissions of these directories will be updated.
-func (w *Writer) CreateWithParents(name string, f *File) error {
+func (w *Writer) MakeParents(name string) error {
 	if err := w.finishInode(); err != nil {
 		return err
 	}
+
 	// go through the directories in the path one by one and create the
 	// parent directories if they don't exist.
 	cleanname := path.Clean("/" + name)[1:]
@@ -553,7 +571,7 @@ func (w *Writer) CreateWithParents(name string, f *File) error {
 		}
 		root = root.Children[dirname]
 	}
-	return w.Create(name, f)
+	return nil
 }
 
 // Create adds a file to the file system.
@@ -603,6 +621,8 @@ func (w *Writer) Create(name string, f *File) error {
 }
 
 // Link adds a hard link to the file system.
+// We support creating hardlinks to symlinks themselves instead of what
+// the symlinks link to, as this is what containerd does upstream.
 func (w *Writer) Link(oldname, newname string) error {
 	if err := w.finishInode(); err != nil {
 		return err
@@ -620,8 +640,8 @@ func (w *Writer) Link(oldname, newname string) error {
 		return err
 	}
 	switch oldfile.Mode & format.TypeMask {
-	case format.S_IFDIR, format.S_IFLNK:
-		return fmt.Errorf("%s: link target cannot be a directory or symlink: %s", newname, oldname)
+	case format.S_IFDIR:
+		return fmt.Errorf("%s: link target cannot be a directory: %s", newname, oldname)
 	}
 
 	if existing != oldfile && oldfile.LinkCount >= format.MaxLinks {

vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/tar2ext4.go

@@ -3,15 +3,14 @@ package tar2ext4
 import (
 	"archive/tar"
 	"bufio"
-	"bytes"
 	"encoding/binary"
+	"fmt"
 	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"strings"
-	"unsafe"
 
 	"github.com/Microsoft/hcsshim/ext4/dmverity"
 	"github.com/Microsoft/hcsshim/ext4/internal/compactext4"
@@ -65,16 +64,16 @@ func MaximumDiskSize(size int64) Option {
 const (
 	whiteoutPrefix = ".wh."
 	opaqueWhiteout = ".wh..wh..opq"
-	ext4blocksize  = compactext4.BlockSize
 )
 
-// Convert writes a compact ext4 file system image that contains the files in the
+// ConvertTarToExt4 writes a compact ext4 file system image that contains the files in the
 // input tar stream.
-func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
+func ConvertTarToExt4(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 	var p params
 	for _, opt := range options {
 		opt(&p)
 	}
+
 	t := tar.NewReader(bufio.NewReader(r))
 	fs := compactext4.NewWriter(w, p.ext4opts...)
 	for {
@@ -86,6 +85,10 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 			return err
 		}
 
+		if err = fs.MakeParents(hdr.Name); err != nil {
+			return errors.Wrapf(err, "failed to ensure parent directories for %s", hdr.Name)
+		}
+
 		if p.convertWhiteout {
 			dir, name := path.Split(hdr.Name)
 			if strings.HasPrefix(name, whiteoutPrefix) {
@@ -93,12 +96,12 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 					// Update the directory with the appropriate xattr.
 					f, err := fs.Stat(dir)
 					if err != nil {
-						return err
+						return errors.Wrapf(err, "failed to stat parent directory of whiteout %s", hdr.Name)
 					}
 					f.Xattrs["trusted.overlay.opaque"] = []byte("y")
 					err = fs.Create(dir, f)
 					if err != nil {
-						return err
+						return errors.Wrapf(err, "failed to create opaque dir %s", hdr.Name)
 					}
 				} else {
 					// Create an overlay-style whiteout.
@@ -109,7 +112,7 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 					}
 					err = fs.Create(path.Join(dir, name[len(whiteoutPrefix):]), f)
 					if err != nil {
-						return err
+						return errors.Wrapf(err, "failed to create whiteout file for %s", hdr.Name)
 					}
 				}
 
@@ -161,7 +164,7 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 			}
 			f.Mode &= ^compactext4.TypeMask
 			f.Mode |= typ
-			err = fs.CreateWithParents(hdr.Name, f)
+			err = fs.Create(hdr.Name, f)
 			if err != nil {
 				return err
 			}
@@ -171,67 +174,29 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 			}
 		}
 	}
-	err := fs.Close()
-	if err != nil {
+	return fs.Close()
+}
+
+// Convert wraps ConvertTarToExt4 and conditionally computes (and appends) the file image's cryptographic
+// hashes (merkle tree) or/and appends a VHD footer.
+func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
+	var p params
+	for _, opt := range options {
+		opt(&p)
+	}
+
+	if err := ConvertTarToExt4(r, w, options...); err != nil {
 		return err
 	}
 
 	if p.appendDMVerity {
-		ext4size, err := w.Seek(0, io.SeekEnd)
-		if err != nil {
-			return err
-		}
-
-		// Rewind the stream and then read it all into a []byte for
-		// dmverity processing
-		_, err = w.Seek(0, io.SeekStart)
-		if err != nil {
-			return err
-		}
-		data, err := ioutil.ReadAll(w)
-		if err != nil {
-			return err
-		}
-
-		mtree, err := dmverity.MerkleTree(data)
-		if err != nil {
-			return errors.Wrap(err, "failed to build merkle tree")
-		}
-
-		// Write dmverity superblock and then the merkle tree after the end of the
-		// ext4 filesystem
-		_, err = w.Seek(0, io.SeekEnd)
-		if err != nil {
-			return err
-		}
-		superblock := dmverity.NewDMVeritySuperblock(uint64(ext4size))
-		err = binary.Write(w, binary.LittleEndian, superblock)
-		if err != nil {
-			return err
-		}
-		// pad the superblock
-		sbsize := int(unsafe.Sizeof(*superblock))
-		padding := bytes.Repeat([]byte{0}, ext4blocksize-(sbsize%ext4blocksize))
-		_, err = w.Write(padding)
-		if err != nil {
-			return err
-		}
-		// write the tree
-		_, err = w.Write(mtree)
-		if err != nil {
+		if err := dmverity.ComputeAndWriteHashDevice(w, w); err != nil {
 			return err
 		}
 	}
 
 	if p.appendVhdFooter {
-		size, err := w.Seek(0, io.SeekEnd)
-		if err != nil {
-			return err
-		}
-		err = binary.Write(w, binary.BigEndian, makeFixedVHDFooter(size))
-		if err != nil {
-			return err
-		}
+		return ConvertToVhd(w)
 	}
 	return nil
 }
@@ -266,5 +231,52 @@ func ReadExt4SuperBlock(vhdPath string) (*format.SuperBlock, error) {
 	if err := binary.Read(vhd, binary.LittleEndian, &sb); err != nil {
 		return nil, err
 	}
+	// Make sure the magic bytes are correct.
+	if sb.Magic != format.SuperBlockMagic {
+		return nil, errors.New("not an ext4 file system")
+	}
 	return &sb, nil
 }
+
+// ConvertAndComputeRootDigest writes a compact ext4 file system image that contains the files in the
+// input tar stream, computes the resulting file image's cryptographic hashes (merkle tree) and returns
+// merkle tree root digest. Convert is called with minimal options: ConvertWhiteout and MaximumDiskSize
+// set to dmverity.RecommendedVHDSizeGB.
+func ConvertAndComputeRootDigest(r io.Reader) (string, error) {
+	out, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temporary file: %s", err)
+	}
+	defer func() {
+		_ = os.Remove(out.Name())
+	}()
+
+	options := []Option{
+		ConvertWhiteout,
+		MaximumDiskSize(dmverity.RecommendedVHDSizeGB),
+	}
+	if err := ConvertTarToExt4(r, out, options...); err != nil {
+		return "", fmt.Errorf("failed to convert tar to ext4: %s", err)
+	}
+
+	if _, err := out.Seek(0, io.SeekStart); err != nil {
+		return "", fmt.Errorf("failed to seek start on temp file when creating merkle tree: %s", err)
+	}
+
+	tree, err := dmverity.MerkleTree(bufio.NewReaderSize(out, dmverity.MerkleTreeBufioSize))
+	if err != nil {
+		return "", fmt.Errorf("failed to create merkle tree: %s", err)
+	}
+
+	hash := dmverity.RootHash(tree)
+	return fmt.Sprintf("%x", hash), nil
+}
+
+// ConvertToVhd converts given io.WriteSeeker to VHD, by appending the VHD footer with a fixed size.
+func ConvertToVhd(w io.WriteSeeker) error {
+	size, err := w.Seek(0, io.SeekEnd)
+	if err != nil {
+		return err
+	}
+	return binary.Write(w, binary.BigEndian, makeFixedVHDFooter(size))
+}