vendor: Bump hcsshim to 0.9.0

This change bumps hcsshim to 0.9.0. Main thing this tag contains is support for
Kubernetes Host Process containers
See: https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/

Signed-off-by: Daniel Canter <dcanter@microsoft.com>

vendor/github.com/Microsoft/hcsshim/ext4/dmverity/dmverity.go

@@ -0,0 +1,198 @@
+package dmverity
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/pkg/errors"
+
+	"github.com/Microsoft/hcsshim/ext4/internal/compactext4"
+)
+
+const (
+	blockSize = compactext4.BlockSize
+	// RecommendedVHDSizeGB is the recommended size in GB for VHDs, which is not a hard limit. 
+	RecommendedVHDSizeGB = 128 * 1024 * 1024 * 1024
+)
+var salt = bytes.Repeat([]byte{0}, 32)
+
+var (
+	ErrSuperBlockReadFailure  = errors.New("failed to read dm-verity super block")
+	ErrSuperBlockParseFailure = errors.New("failed to parse dm-verity super block")
+	ErrRootHashReadFailure    = errors.New("failed to read dm-verity root hash")
+)
+
+type dmveritySuperblock struct {
+	/* (0) "verity\0\0" */
+	Signature [8]byte
+	/* (8) superblock version, 1 */
+	Version uint32
+	/* (12) 0 - Chrome OS, 1 - normal */
+	HashType uint32
+	/* (16) UUID of hash device */
+	UUID [16]byte
+	/* (32) Name of the hash algorithm (e.g., sha256) */
+	Algorithm [32]byte
+	/* (64) The data block size in bytes */
+	DataBlockSize uint32
+	/* (68) The hash block size in bytes */
+	HashBlockSize uint32
+	/* (72) The number of data blocks */
+	DataBlocks uint64
+	/* (80) Size of the salt */
+	SaltSize uint16
+	/* (82) Padding */
+	_ [6]byte
+	/* (88) The salt */
+	Salt [256]byte
+	/* (344) Padding */
+	_ [168]byte
+}
+
+// VerityInfo is minimal exported version of dmveritySuperblock
+type VerityInfo struct {
+	// Offset in blocks on hash device
+	HashOffsetInBlocks int64
+	// Set to true, when dm-verity super block is also written on the hash device
+	SuperBlock    bool
+	RootDigest    string
+	Salt          string
+	Algorithm     string
+	DataBlockSize uint32
+	HashBlockSize uint32
+	DataBlocks    uint64
+	Version       uint32
+}
+
+// MerkleTree constructs dm-verity hash-tree for a given byte array with a fixed salt (0-byte) and algorithm (sha256).
+func MerkleTree(data []byte) ([]byte, error) {
+	layers := make([][]byte, 0)
+
+	currentLevel := bytes.NewBuffer(data)
+
+	for currentLevel.Len() != blockSize {
+		blocks := currentLevel.Len() / blockSize
+		nextLevel := bytes.NewBuffer(make([]byte, 0))
+
+		for i := 0; i < blocks; i++ {
+			block := make([]byte, blockSize)
+			_, err := currentLevel.Read(block)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to read data block")
+			}
+			h := hash2(salt, block)
+			nextLevel.Write(h)
+		}
+
+		padding := bytes.Repeat([]byte{0}, blockSize-(nextLevel.Len()%blockSize))
+		nextLevel.Write(padding)
+
+		currentLevel = nextLevel
+		layers = append(layers, currentLevel.Bytes())
+	}
+
+	var tree = bytes.NewBuffer(make([]byte, 0))
+	for i := len(layers) - 1; i >= 0; i-- {
+		_, err := tree.Write(layers[i])
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to write merkle tree")
+		}
+	}
+
+	return tree.Bytes(), nil
+}
+
+// RootHash computes root hash of dm-verity hash-tree
+func RootHash(tree []byte) []byte {
+	return hash2(salt, tree[:blockSize])
+}
+
+// NewDMVeritySuperblock returns a dm-verity superblock for a device with a given size, salt, algorithm and versions are
+// fixed.
+func NewDMVeritySuperblock(size uint64) *dmveritySuperblock {
+	superblock := &dmveritySuperblock{
+		Version:       1,
+		HashType:      1,
+		UUID:          generateUUID(),
+		DataBlockSize: blockSize,
+		HashBlockSize: blockSize,
+		DataBlocks:    size / blockSize,
+		SaltSize:      uint16(len(salt)),
+	}
+
+	copy(superblock.Signature[:], "verity")
+	copy(superblock.Algorithm[:], "sha256")
+	copy(superblock.Salt[:], salt)
+
+	return superblock
+}
+
+func hash2(a, b []byte) []byte {
+	h := sha256.New()
+	h.Write(append(a, b...))
+	return h.Sum(nil)
+}
+
+func generateUUID() [16]byte {
+	res := [16]byte{}
+	if _, err := rand.Read(res[:]); err != nil {
+		panic(err)
+	}
+	return res
+}
+
+// ReadDMVerityInfo extracts dm-verity super block information and merkle tree root hash
+func ReadDMVerityInfo(vhdPath string, offsetInBytes int64) (*VerityInfo, error) {
+	vhd, err := os.OpenFile(vhdPath, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	defer vhd.Close()
+
+	// Skip the ext4 data to get to dm-verity super block
+	if s, err := vhd.Seek(offsetInBytes, io.SeekStart); err != nil || s != offsetInBytes {
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to seek dm-verity super block")
+		}
+		return nil, errors.Errorf("failed to seek dm-verity super block: expected bytes=%d, actual=%d", offsetInBytes, s)
+	}
+
+	block := make([]byte, blockSize)
+	if s, err := vhd.Read(block); err != nil || s != blockSize {
+		if err != nil {
+			return nil, errors.Wrapf(ErrSuperBlockReadFailure, "%s", err)
+		}
+		return nil, errors.Wrapf(ErrSuperBlockReadFailure, "unexpected bytes read: expected=%d, actual=%d", blockSize, s)
+	}
+
+	dmvSB := &dmveritySuperblock{}
+	b := bytes.NewBuffer(block)
+	if err := binary.Read(b, binary.LittleEndian, dmvSB); err != nil {
+		return nil, errors.Wrapf(ErrSuperBlockParseFailure, "%s", err)
+	}
+
+	// read the merkle tree root
+	if s, err := vhd.Read(block); err != nil || s != blockSize {
+		if err != nil {
+			return nil, errors.Wrapf(ErrRootHashReadFailure, "%s", err)
+		}
+		return nil, errors.Wrapf(ErrRootHashReadFailure, "unexpected bytes read: expected=%d, actual=%d", blockSize, s)
+	}
+	rootHash := hash2(dmvSB.Salt[:dmvSB.SaltSize], block)
+	return &VerityInfo{
+		RootDigest:         fmt.Sprintf("%x", rootHash),
+		Algorithm:          string(bytes.Trim(dmvSB.Algorithm[:], "\x00")),
+		Salt:               fmt.Sprintf("%x", dmvSB.Salt[:dmvSB.SaltSize]),
+		HashOffsetInBlocks: int64(dmvSB.DataBlocks),
+		SuperBlock:         true,
+		DataBlocks:         dmvSB.DataBlocks,
+		DataBlockSize:      dmvSB.DataBlockSize,
+		HashBlockSize:      blockSize,
+		Version:            dmvSB.Version,
+	}, nil
+}

vendor/github.com/Microsoft/hcsshim/ext4/internal/compactext4/compact.go

@@ -95,17 +95,17 @@ const (
 	inodeFirst        = 11
 	inodeLostAndFound = inodeFirst
 
-	blockSize               = 4096
-	blocksPerGroup          = blockSize * 8
+	BlockSize               = 4096
+	blocksPerGroup          = BlockSize * 8
 	inodeSize               = 256
-	maxInodesPerGroup       = blockSize * 8 // Limited by the inode bitmap
-	inodesPerGroupIncrement = blockSize / inodeSize
+	maxInodesPerGroup       = BlockSize * 8 // Limited by the inode bitmap
+	inodesPerGroupIncrement = BlockSize / inodeSize
 
 	defaultMaxDiskSize = 16 * 1024 * 1024 * 1024        // 16GB
 	maxMaxDiskSize     = 16 * 1024 * 1024 * 1024 * 1024 // 16TB
 
 	groupDescriptorSize      = 32 // Use the small group descriptor
-	groupsPerDescriptorBlock = blockSize / groupDescriptorSize
+	groupsPerDescriptorBlock = BlockSize / groupDescriptorSize
 
 	maxFileSize             = 128 * 1024 * 1024 * 1024 // 128GB file size maximum for now
 	smallSymlinkSize        = 59                       // max symlink size that goes directly in the inode
@@ -252,7 +252,7 @@ type xattrState struct {
 
 func (s *xattrState) init() {
 	s.inodeLeft = inodeExtraSize - xattrInodeOverhead
-	s.blockLeft = blockSize - xattrBlockOverhead
+	s.blockLeft = BlockSize - xattrBlockOverhead
 }
 
 func (s *xattrState) addXattr(name string, value []byte) bool {
@@ -331,7 +331,7 @@ func (w *Writer) writeXattrs(inode *inode, state *xattrState) error {
 				state.block[i].Name < state.block[j].Name
 		})
 
-		var b [blockSize]byte
+		var b [BlockSize]byte
 		binary.LittleEndian.PutUint32(b[0:], format.XAttrHeaderMagic) // Magic
 		binary.LittleEndian.PutUint32(b[4:], 1)                       // ReferenceCount
 		binary.LittleEndian.PutUint32(b[8:], 1)                       // Blocks
@@ -665,7 +665,7 @@ func (w *Writer) Stat(name string) (*File, error) {
 			if w.err != nil {
 				return nil, w.err
 			}
-			var b [blockSize]byte
+			var b [BlockSize]byte
 			_, err := w.f.Read(b[:])
 			w.seekBlock(orig)
 			if err != nil {
@@ -717,11 +717,11 @@ func (w *Writer) startInode(name string, inode *inode, size int64) {
 }
 
 func (w *Writer) block() uint32 {
-	return uint32(w.pos / blockSize)
+	return uint32(w.pos / BlockSize)
 }
 
 func (w *Writer) seekBlock(block uint32) {
-	w.pos = int64(block) * blockSize
+	w.pos = int64(block) * BlockSize
 	if w.err != nil {
 		return
 	}
@@ -733,9 +733,9 @@ func (w *Writer) seekBlock(block uint32) {
 }
 
 func (w *Writer) nextBlock() {
-	if w.pos%blockSize != 0 {
+	if w.pos%BlockSize != 0 {
 		// Simplify callers; w.err is updated on failure.
-		_, _ = w.zero(blockSize - w.pos%blockSize)
+		_, _ = w.zero(BlockSize - w.pos%BlockSize)
 	}
 }
 
@@ -763,17 +763,17 @@ func fillExtents(hdr *format.ExtentHeader, extents []format.ExtentLeafNode, star
 
 func (w *Writer) writeExtents(inode *inode) error {
 	start := w.pos - w.dataWritten
-	if start%blockSize != 0 {
+	if start%BlockSize != 0 {
 		panic("unaligned")
 	}
 	w.nextBlock()
 
-	startBlock := uint32(start / blockSize)
+	startBlock := uint32(start / BlockSize)
 	blocks := w.block() - startBlock
 	usedBlocks := blocks
 
 	const extentNodeSize = 12
-	const extentsPerBlock = blockSize/extentNodeSize - 1
+	const extentsPerBlock = BlockSize/extentNodeSize - 1
 
 	extents := (blocks + maxBlocksPerExtent - 1) / maxBlocksPerExtent
 	var b bytes.Buffer
@@ -787,7 +787,7 @@ func (w *Writer) writeExtents(inode *inode) error {
 		fillExtents(&root.hdr, root.extents[:extents], startBlock, 0, blocks)
 		_ = binary.Write(&b, binary.LittleEndian, root)
 	} else if extents <= 4*extentsPerBlock {
-		const extentsPerBlock = blockSize/extentNodeSize - 1
+		const extentsPerBlock = BlockSize/extentNodeSize - 1
 		extentBlocks := extents/extentsPerBlock + 1
 		usedBlocks += extentBlocks
 		var b2 bytes.Buffer
@@ -815,13 +815,13 @@ func (w *Writer) writeExtents(inode *inode) error {
 			var node struct {
 				hdr     format.ExtentHeader
 				extents [extentsPerBlock]format.ExtentLeafNode
-				_       [blockSize - (extentsPerBlock+1)*extentNodeSize]byte
+				_       [BlockSize - (extentsPerBlock+1)*extentNodeSize]byte
 			}
 
 			offset := i * extentsPerBlock * maxBlocksPerExtent
 			fillExtents(&node.hdr, node.extents[:extentsInBlock], startBlock+offset, offset, blocks)
 			_ = binary.Write(&b2, binary.LittleEndian, node)
-			if _, err := w.write(b2.Next(blockSize)); err != nil {
+			if _, err := w.write(b2.Next(BlockSize)); err != nil {
 				return err
 			}
 		}
@@ -900,7 +900,7 @@ func (w *Writer) writeDirectory(dir, parent *inode) error {
 
 	// The size of the directory is not known yet.
 	w.startInode("", dir, 0x7fffffffffffffff)
-	left := blockSize
+	left := BlockSize
 	finishBlock := func() error {
 		if left > 0 {
 			e := format.DirectoryEntry{
@@ -919,7 +919,7 @@ func (w *Writer) writeDirectory(dir, parent *inode) error {
 				return err
 			}
 		}
-		left = blockSize
+		left = BlockSize
 		return nil
 	}
 
@@ -1114,7 +1114,7 @@ func MaximumDiskSize(size int64) Option {
 		} else if size == 0 {
 			w.maxDiskSize = defaultMaxDiskSize
 		} else {
-			w.maxDiskSize = (size + blockSize - 1) &^ (blockSize - 1)
+			w.maxDiskSize = (size + BlockSize - 1) &^ (BlockSize - 1)
 		}
 	}
 }
@@ -1129,7 +1129,7 @@ func (w *Writer) init() error {
 	root.LinkCount++ // The root is linked to itself.
 	// Skip until the first non-reserved inode.
 	w.inodes = append(w.inodes, make([]*inode, inodeFirst-len(w.inodes)-1)...)
-	maxBlocks := (w.maxDiskSize-1)/blockSize + 1
+	maxBlocks := (w.maxDiskSize-1)/BlockSize + 1
 	maxGroups := (maxBlocks-1)/blocksPerGroup + 1
 	w.gdBlocks = uint32((maxGroups-1)/groupsPerDescriptorBlock + 1)
 
@@ -1145,7 +1145,7 @@ func (w *Writer) init() error {
 }
 
 func groupCount(blocks uint32, inodes uint32, inodesPerGroup uint32) uint32 {
-	inodeBlocksPerGroup := inodesPerGroup * inodeSize / blockSize
+	inodeBlocksPerGroup := inodesPerGroup * inodeSize / BlockSize
 	dataBlocksPerGroup := blocksPerGroup - inodeBlocksPerGroup - 2 // save room for the bitmaps
 
 	// Increase the block count to ensure there are enough groups for all the
@@ -1207,16 +1207,16 @@ func (w *Writer) Close() error {
 	}
 
 	gds := make([]format.GroupDescriptor, w.gdBlocks*groupsPerDescriptorBlock)
-	inodeTableSizePerGroup := inodesPerGroup * inodeSize / blockSize
+	inodeTableSizePerGroup := inodesPerGroup * inodeSize / BlockSize
 	var totalUsedBlocks, totalUsedInodes uint32
 	for g := uint32(0); g < groups; g++ {
-		var b [blockSize * 2]byte
+		var b [BlockSize * 2]byte
 		var dirCount, usedInodeCount, usedBlockCount uint16
 
 		// Block bitmap
 		if (g+1)*blocksPerGroup <= validDataSize {
 			// This group is fully allocated.
-			for j := range b[:blockSize] {
+			for j := range b[:BlockSize] {
 				b[j] = 0xff
 			}
 			usedBlockCount = blocksPerGroup
@@ -1246,7 +1246,7 @@ func (w *Writer) Close() error {
 			ino := format.InodeNumber(1 + g*inodesPerGroup + j)
 			inode := w.getInode(ino)
 			if ino < inodeFirst || inode != nil {
-				b[blockSize+j/8] |= 1 << (j % 8)
+				b[BlockSize+j/8] |= 1 << (j % 8)
 				usedInodeCount++
 			}
 			if inode != nil && inode.Mode&format.TypeMask == format.S_IFDIR {
@@ -1271,7 +1271,7 @@ func (w *Writer) Close() error {
 	}
 
 	// Zero up to the disk size.
-	_, err = w.zero(int64(diskSize-bitmapOffset-bitmapSize) * blockSize)
+	_, err = w.zero(int64(diskSize-bitmapOffset-bitmapSize) * BlockSize)
 	if err != nil {
 		return err
 	}
@@ -1287,7 +1287,7 @@ func (w *Writer) Close() error {
 	}
 
 	// Write the super block
-	var blk [blockSize]byte
+	var blk [BlockSize]byte
 	b := bytes.NewBuffer(blk[:1024])
 	sb := &format.SuperBlock{
 		InodesCount:        inodesPerGroup * groups,

vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/tar2ext4.go

@@ -3,12 +3,17 @@ package tar2ext4
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
 	"encoding/binary"
+	"github.com/pkg/errors"
 	"io"
+	"io/ioutil"
 	"os"
 	"path"
 	"strings"
+	"unsafe"
 
+	"github.com/Microsoft/hcsshim/ext4/dmverity"
 	"github.com/Microsoft/hcsshim/ext4/internal/compactext4"
 	"github.com/Microsoft/hcsshim/ext4/internal/format"
 )
@@ -16,6 +21,7 @@ import (
 type params struct {
 	convertWhiteout bool
 	appendVhdFooter bool
+	appendDMVerity  bool
 	ext4opts        []compactext4.Option
 }
 
@@ -34,6 +40,12 @@ func AppendVhdFooter(p *params) {
 	p.appendVhdFooter = true
 }
 
+// AppendDMVerity instructs the converter to add a dmverity merkle tree for
+// the ext4 filesystem after the filesystem and before the optional VHD footer
+func AppendDMVerity(p *params) {
+	p.appendDMVerity = true
+}
+
 // InlineData instructs the converter to write small files into the inode
 // structures directly. This creates smaller images but currently is not
 // compatible with DAX.
@@ -53,6 +65,7 @@ func MaximumDiskSize(size int64) Option {
 const (
 	whiteoutPrefix = ".wh."
 	opaqueWhiteout = ".wh..wh..opq"
+	ext4blocksize  = compactext4.BlockSize
 )
 
 // Convert writes a compact ext4 file system image that contains the files in the
@@ -162,6 +175,54 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 	if err != nil {
 		return err
 	}
+
+	if p.appendDMVerity {
+		ext4size, err := w.Seek(0, io.SeekEnd)
+		if err != nil {
+			return err
+		}
+
+		// Rewind the stream and then read it all into a []byte for
+		// dmverity processing
+		_, err = w.Seek(0, io.SeekStart)
+		if err != nil {
+			return err
+		}
+		data, err := ioutil.ReadAll(w)
+		if err != nil {
+			return err
+		}
+
+		mtree, err := dmverity.MerkleTree(data)
+		if err != nil {
+			return errors.Wrap(err, "failed to build merkle tree")
+		}
+
+		// Write dmverity superblock and then the merkle tree after the end of the
+		// ext4 filesystem
+		_, err = w.Seek(0, io.SeekEnd)
+		if err != nil {
+			return err
+		}
+		superblock := dmverity.NewDMVeritySuperblock(uint64(ext4size))
+		err = binary.Write(w, binary.LittleEndian, superblock)
+		if err != nil {
+			return err
+		}
+		// pad the superblock
+		sbsize := int(unsafe.Sizeof(*superblock))
+		padding := bytes.Repeat([]byte{0}, ext4blocksize-(sbsize%ext4blocksize))
+		_, err = w.Write(padding)
+		if err != nil {
+			return err
+		}
+		// write the tree
+		_, err = w.Write(mtree)
+		if err != nil {
+			return err
+		}
+	}
+
 	if p.appendVhdFooter {
 		size, err := w.Seek(0, io.SeekEnd)
 		if err != nil {