github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Use go-winio tar-application code instead of our own

Browse Source 
applyFunc now takes an io.Reader instead of a tar.Reader because I'm
trying to mirror the API of the not-yet-exposed implementation of this
same behaviour in github.com/Microsoft/hcsshim/internal/ociwclayer,
with an eye to later moving to that implementation it is ever exposed.

Signed-off-by: Paul "TBBle" Hampson <Paul.Hampson@Pobox.com>
This commit is contained in:
Paul "TBBle" Hampson
2020-07-16 23:24:07 +10:00
parent 05647251a5
commit 3e47cdf70f
8 changed files with 496 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
archive/strconv.go
View File

@@ -1,68 +0,0 @@
// +build windows
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package archive
import (
"strconv"
"strings"
"time"
"archive/tar"
)
// Forked from https://github.com/golang/go/blob/master/src/archive/tar/strconv.go
// as archive/tar doesn't support CreationTime, but does handle PAX time parsing,
// and there's no need to re-invent the wheel.
// parsePAXTime takes a string of the form %d.%d as described in the PAX
// specification. Note that this implementation allows for negative timestamps,
// which is allowed for by the PAX specification, but not always portable.
func parsePAXTime(s string) (time.Time, error) {
const maxNanoSecondDigits = 9
// Split string into seconds and sub-seconds parts.
ss, sn := s, ""
if pos := strings.IndexByte(s, '.'); pos >= 0 {
ss, sn = s[:pos], s[pos+1:]
}
// Parse the seconds.
secs, err := strconv.ParseInt(ss, 10, 64)
if err != nil {
return time.Time{}, tar.ErrHeader
}
if len(sn) == 0 {
return time.Unix(secs, 0), nil // No sub-second values
}
// Parse the nanoseconds.
if strings.Trim(sn, "0123456789") != "" {
return time.Time{}, tar.ErrHeader
}
if len(sn) < maxNanoSecondDigits {
sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad
} else {
sn = sn[:maxNanoSecondDigits] // Right truncate
}
nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
if len(ss) > 0 && ss[0] == '-' {
return time.Unix(secs, -nsecs), nil // Negative correction
}
return time.Unix(secs, nsecs), nil
}

6
archive/tar.go
View File

@@ -114,16 +114,18 @@ func Apply(ctx context.Context, root string, r io.Reader, opts ...ApplyOpt) (int
options.applyFunc = applyNaive
}
return options.applyFunc(ctx, root, tar.NewReader(r), options)
return options.applyFunc(ctx, root, r, options)
}
// applyNaive applies a tar stream of an OCI style diff tar to a directory
// applying each file as either a whole file or whiteout.
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyOptions) (size int64, err error) {
func applyNaive(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) {
var (
dirs []*tar.Header
tr = tar.NewReader(r)
// Used for handling opaque directory markers which
// may occur out of order
unpackedPaths = make(map[string]struct{})

3
archive/tar_opts.go
View File

@@ -19,6 +19,7 @@ package archive
import (
"archive/tar"
"context"
"io"
)
// ApplyOptions provides additional options for an Apply operation
@@ -27,7 +28,7 @@ type ApplyOptions struct {
ConvertWhiteout ConvertWhiteout // Convert whiteout files
Parents []string // Parent directories to handle inherited attributes without CoW
applyFunc func(context.Context, string, *tar.Reader, ApplyOptions) (int64, error)
applyFunc func(context.Context, string, io.Reader, ApplyOptions) (int64, error)
}
// ApplyOpt allows setting mutable archive apply properties on creation

193
archive/tar_windows.go
View File

@@ -22,38 +22,20 @@ import (
"archive/tar"
"bufio"
"context"
"encoding/base64"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"syscall"
"github.com/Microsoft/go-winio"
"github.com/Microsoft/go-winio/backuptar"
"github.com/Microsoft/hcsshim"
"github.com/containerd/containerd/sys"
"github.com/pkg/errors"
)
const (
// MSWINDOWS pax vendor extensions
hdrMSWindowsPrefix = "MSWINDOWS."
hdrFileAttributes = hdrMSWindowsPrefix + "fileattr"
hdrSecurityDescriptor = hdrMSWindowsPrefix + "sd"
hdrRawSecurityDescriptor = hdrMSWindowsPrefix + "rawsd"
hdrMountPoint = hdrMSWindowsPrefix + "mountpoint"
hdrEaPrefix = hdrMSWindowsPrefix + "xattr."
// LIBARCHIVE pax vendor extensions
hdrLibArchivePrefix = "LIBARCHIVE."
hdrCreateTime = hdrLibArchivePrefix + "creationtime"
)
var (
// mutatedFiles is a list of files that are mutated by the import process
// and must be backed up and restored.
@@ -152,7 +134,7 @@ func setxattr(path, key, value string) error {
// applyWindowsLayer applies a tar stream of an OCI style diff tar of a Windows
// layer using the hcsshim layer writer and backup streams.
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options ApplyOptions) (size int64, err error) {
func applyWindowsLayer(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) {
home, id := filepath.Split(root)
info := hcsshim.DriverInfo{
HomeDir: home,
@@ -172,6 +154,7 @@ func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options
}
}()
tr := tar.NewReader(r)
buf := bufio.NewWriter(nil)
hdr, nextErr := tr.Next()
// Iterate through the files in the archive.
@@ -208,7 +191,7 @@ func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options
}
hdr, nextErr = tr.Next()
} else {
name, fileSize, fileInfo, err := fileInfoFromHeader(hdr)
name, fileSize, fileInfo, err := backuptar.FileInfoFromHeader(hdr)
if err != nil {
return 0, err
}
@@ -223,42 +206,6 @@ func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options
return
}
// fileInfoFromHeader retrieves basic Win32 file information from a tar header, using the additional metadata written by
// WriteTarFileFromBackupStream.
func fileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) {
name = hdr.Name
if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
size = hdr.Size
}
fileInfo = &winio.FileBasicInfo{
LastAccessTime: syscall.NsecToFiletime(hdr.AccessTime.UnixNano()),
LastWriteTime: syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
ChangeTime: syscall.NsecToFiletime(hdr.ChangeTime.UnixNano()),
// Default CreationTime to ModTime, updated below if MSWINDOWS.createtime exists
CreationTime: syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
}
if attrStr, ok := hdr.PAXRecords[hdrFileAttributes]; ok {
attr, err := strconv.ParseUint(attrStr, 10, 32)
if err != nil {
return "", 0, nil, err
}
fileInfo.FileAttributes = uint32(attr)
} else {
if hdr.Typeflag == tar.TypeDir {
fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY
}
}
if createStr, ok := hdr.PAXRecords[hdrCreateTime]; ok {
createTime, err := parsePAXTime(createStr)
if err != nil {
return "", 0, nil, err
}
fileInfo.CreationTime = syscall.NsecToFiletime(createTime.UnixNano())
}
return
}
// tarToBackupStreamWithMutatedFiles reads data from a tar stream and
// writes it to a backup stream, and also saves any files that will be mutated
// by the import layer process to a backup location.
@@ -299,137 +246,7 @@ func tarToBackupStreamWithMutatedFiles(buf *bufio.Writer, w io.Writer, t *tar.Re
}
}()
return writeBackupStreamFromTarFile(buf, t, hdr)
}
// writeBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
// tar file entries in order to collect all the alternate data streams for the file, it returns the next
// tar file that was not processed, or io.EOF is there are no more.
func writeBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) {
bw := winio.NewBackupStreamWriter(w)
var sd []byte
var err error
// Maintaining old SDDL-based behavior for backward compatibility. All new tar headers written
// by this library will have raw binary for the security descriptor.
if sddl, ok := hdr.PAXRecords[hdrSecurityDescriptor]; ok {
sd, err = winio.SddlToSecurityDescriptor(sddl)
if err != nil {
return nil, err
}
}
if sdraw, ok := hdr.PAXRecords[hdrRawSecurityDescriptor]; ok {
sd, err = base64.StdEncoding.DecodeString(sdraw)
if err != nil {
return nil, err
}
}
if len(sd) != 0 {
bhdr := winio.BackupHeader{
Id: winio.BackupSecurity,
Size: int64(len(sd)),
}
err := bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = bw.Write(sd)
if err != nil {
return nil, err
}
}
var eas []winio.ExtendedAttribute
for k, v := range hdr.PAXRecords {
if !strings.HasPrefix(k, hdrEaPrefix) {
continue
}
data, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return nil, err
}
eas = append(eas, winio.ExtendedAttribute{
Name: k[len(hdrEaPrefix):],
Value: data,
})
}
if len(eas) != 0 {
eadata, err := winio.EncodeExtendedAttributes(eas)
if err != nil {
return nil, err
}
bhdr := winio.BackupHeader{
Id: winio.BackupEaData,
Size: int64(len(eadata)),
}
err = bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = bw.Write(eadata)
if err != nil {
return nil, err
}
}
if hdr.Typeflag == tar.TypeSymlink {
_, isMountPoint := hdr.PAXRecords[hdrMountPoint]
rp := winio.ReparsePoint{
Target: filepath.FromSlash(hdr.Linkname),
IsMountPoint: isMountPoint,
}
reparse := winio.EncodeReparsePoint(&rp)
bhdr := winio.BackupHeader{
Id: winio.BackupReparseData,
Size: int64(len(reparse)),
}
err := bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = bw.Write(reparse)
if err != nil {
return nil, err
}
}
buf := bufPool.Get().(*[]byte)
defer bufPool.Put(buf)
if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
bhdr := winio.BackupHeader{
Id: winio.BackupData,
Size: hdr.Size,
}
err := bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = io.CopyBuffer(bw, t, *buf)
if err != nil {
return nil, err
}
}
// Copy all the alternate data streams and return the next non-ADS header.
for {
ahdr, err := t.Next()
if err != nil {
return nil, err
}
if ahdr.Typeflag != tar.TypeReg || !strings.HasPrefix(ahdr.Name, hdr.Name+":") {
return ahdr, nil
}
bhdr := winio.BackupHeader{
Id: winio.BackupAlternateData,
Size: ahdr.Size,
Name: ahdr.Name[len(hdr.Name):] + ":$DATA",
}
err = bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = io.CopyBuffer(bw, t, *buf)
if err != nil {
return nil, err
}
}
return backuptar.WriteBackupStreamFromTarFile(buf, t, hdr)
}
func copyDirInfo(fi os.FileInfo, path string) error {