diff --git a/archive/tar.go b/archive/tar.go index ebdbb5f24..0fcfb6539 100644 --- a/archive/tar.go +++ b/archive/tar.go @@ -63,13 +63,34 @@ func Diff(ctx context.Context, a, b string) io.ReadCloser { } // WriteDiff writes a tar stream of the computed difference between the -// provided directories. +// provided paths. // // Produces a tar using OCI style file markers for deletions. Deleted // files will be prepended with the prefix ".wh.". This style is // based off AUFS whiteouts. // See https://github.com/opencontainers/image-spec/blob/master/layer.md -func WriteDiff(ctx context.Context, w io.Writer, a, b string) error { +func WriteDiff(ctx context.Context, w io.Writer, a, b string, opts ...WriteDiffOpt) error { + var options WriteDiffOptions + for _, opt := range opts { + if err := opt(&options); err != nil { + return errors.Wrap(err, "failed to apply option") + } + } + if options.writeDiffFunc == nil { + options.writeDiffFunc = writeDiffNaive + } + + return options.writeDiffFunc(ctx, w, a, b, options) +} + +// writeDiffNaive writes a tar stream of the computed difference between the +// provided directories on disk. +// +// Produces a tar using OCI style file markers for deletions. Deleted +// files will be prepended with the prefix ".wh.". This style is +// based off AUFS whiteouts. +// See https://github.com/opencontainers/image-spec/blob/master/layer.md +func writeDiffNaive(ctx context.Context, w io.Writer, a, b string, _ WriteDiffOptions) error { cw := newChangeWriter(w, b) err := fs.Changes(ctx, a, b, cw.HandleChange) if err != nil { diff --git a/archive/tar_linux_test.go b/archive/tar_linux_test.go index f0984681a..2ef8d43a7 100644 --- a/archive/tar_linux_test.go +++ b/archive/tar_linux_test.go @@ -70,7 +70,7 @@ func TestOverlayApplyNoParents(t *testing.T) { } fstest.FSSuite(t, overlayDiffApplier{ tmp: base, - diff: func(ctx context.Context, w io.Writer, a, b string) error { + diff: func(ctx context.Context, w io.Writer, a, b string, _ ...WriteDiffOpt) error { cw := newChangeWriter(w, b) cw.addedDirs = nil err := fs.Changes(ctx, a, b, cw.HandleChange) @@ -85,7 +85,7 @@ func TestOverlayApplyNoParents(t *testing.T) { type overlayDiffApplier struct { tmp string - diff func(context.Context, io.Writer, string, string) error + diff func(context.Context, io.Writer, string, string, ...WriteDiffOpt) error t *testing.T } diff --git a/archive/tar_opts.go b/archive/tar_opts.go index 9f058f3a6..58985555a 100644 --- a/archive/tar_opts.go +++ b/archive/tar_opts.go @@ -73,3 +73,13 @@ func WithParents(p []string) ApplyOpt { return nil } } + +// WriteDiffOptions provides additional options for a WriteDiff operation +type WriteDiffOptions struct { + ParentLayers []string // Windows needs the full list of parent layers + + writeDiffFunc func(context.Context, io.Writer, string, string, WriteDiffOptions) error +} + +// WriteDiffOpt allows setting mutable archive write properties on creation +type WriteDiffOpt func(options *WriteDiffOptions) error diff --git a/archive/tar_opts_windows.go b/archive/tar_opts_windows.go index f472013bc..d3b8f4fb9 100644 --- a/archive/tar_opts_windows.go +++ b/archive/tar_opts_windows.go @@ -18,6 +18,19 @@ package archive +import ( + "context" + "io" + + "github.com/Microsoft/hcsshim/pkg/ociwclayer" +) + +// applyWindowsLayer applies a tar stream of an OCI style diff tar of a Windows layer +// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets +func applyWindowsLayer(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) { + return ociwclayer.ImportLayerFromTar(ctx, r, root, options.Parents) +} + // AsWindowsContainerLayer indicates that the tar stream to apply is that of // a Windows Container Layer. The caller must be holding SeBackupPrivilege and // SeRestorePrivilege. @@ -27,3 +40,33 @@ func AsWindowsContainerLayer() ApplyOpt { return nil } } + +// writeDiffWindowsLayers writes a tar stream of the computed difference between the +// provided Windows layers +// +// Produces a tar using OCI style file markers for deletions. Deleted +// files will be prepended with the prefix ".wh.". This style is +// based off AUFS whiteouts. +// See https://github.com/opencontainers/image-spec/blob/master/layer.md +func writeDiffWindowsLayers(ctx context.Context, w io.Writer, _, layer string, options WriteDiffOptions) error { + return ociwclayer.ExportLayerToTar(ctx, w, layer, options.ParentLayers) +} + +// AsWindowsContainerLayerPair indicates that the paths to diff are a pair of +// Windows Container Layers. The caller must be holding SeBackupPrivilege. +func AsWindowsContainerLayerPair() WriteDiffOpt { + return func(options *WriteDiffOptions) error { + options.writeDiffFunc = writeDiffWindowsLayers + return nil + } +} + +// WithParentLayers provides the Windows Container Layers that are the parents +// of the target (right-hand, "upper") layer, if any. The source (left-hand, "lower") +// layer passed to WriteDiff should be "" in this case. +func WithParentLayers(p []string) WriteDiffOpt { + return func(options *WriteDiffOptions) error { + options.ParentLayers = p + return nil + } +} diff --git a/archive/tar_windows.go b/archive/tar_windows.go index d8f50845e..e30229f7e 100644 --- a/archive/tar_windows.go +++ b/archive/tar_windows.go @@ -20,33 +20,14 @@ package archive import ( "archive/tar" - "bufio" - "context" "fmt" - "io" "os" - "path" - "path/filepath" "strings" - "github.com/Microsoft/go-winio" - "github.com/Microsoft/go-winio/backuptar" - "github.com/Microsoft/hcsshim" "github.com/containerd/containerd/sys" "github.com/pkg/errors" ) -var ( - // mutatedFiles is a list of files that are mutated by the import process - // and must be backed up and restored. - mutatedFiles = map[string]string{ - "UtilityVM/Files/EFI/Microsoft/Boot/BCD": "bcd.bak", - "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG": "bcd.log.bak", - "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG1": "bcd.log1.bak", - "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG2": "bcd.log2.bak", - } -) - // tarName returns platform-specific filepath // to canonical posix-style path for tar archival. p is relative // path. @@ -141,121 +122,3 @@ func copyDirInfo(fi os.FileInfo, path string) error { func copyUpXAttrs(dst, src string) error { return nil } - -// applyWindowsLayer applies a tar stream of an OCI style diff tar of a Windows -// layer using the hcsshim layer writer and backup streams. -// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets -func applyWindowsLayer(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) { - home, id := filepath.Split(root) - info := hcsshim.DriverInfo{ - HomeDir: home, - } - - w, err := hcsshim.NewLayerWriter(info, id, options.Parents) - if err != nil { - return 0, err - } - defer func() { - if err2 := w.Close(); err2 != nil { - // This error should not be discarded as a failure here - // could result in an invalid layer on disk - if err == nil { - err = err2 - } - } - }() - - tr := tar.NewReader(r) - buf := bufio.NewWriter(nil) - hdr, nextErr := tr.Next() - // Iterate through the files in the archive. - for { - select { - case <-ctx.Done(): - return 0, ctx.Err() - default: - } - - if nextErr == io.EOF { - // end of tar archive - break - } - if nextErr != nil { - return 0, nextErr - } - - // Note: path is used instead of filepath to prevent OS specific handling - // of the tar path - base := path.Base(hdr.Name) - if strings.HasPrefix(base, whiteoutPrefix) { - dir := path.Dir(hdr.Name) - originalBase := base[len(whiteoutPrefix):] - originalPath := path.Join(dir, originalBase) - if err := w.Remove(filepath.FromSlash(originalPath)); err != nil { - return 0, err - } - hdr, nextErr = tr.Next() - } else if hdr.Typeflag == tar.TypeLink { - err := w.AddLink(filepath.FromSlash(hdr.Name), filepath.FromSlash(hdr.Linkname)) - if err != nil { - return 0, err - } - hdr, nextErr = tr.Next() - } else { - name, fileSize, fileInfo, err := backuptar.FileInfoFromHeader(hdr) - if err != nil { - return 0, err - } - if err := w.Add(filepath.FromSlash(name), fileInfo); err != nil { - return 0, err - } - size += fileSize - hdr, nextErr = tarToBackupStreamWithMutatedFiles(buf, w, tr, hdr, root) - } - } - - return -} - -// tarToBackupStreamWithMutatedFiles reads data from a tar stream and -// writes it to a backup stream, and also saves any files that will be mutated -// by the import layer process to a backup location. -func tarToBackupStreamWithMutatedFiles(buf *bufio.Writer, w io.Writer, t *tar.Reader, hdr *tar.Header, root string) (nextHdr *tar.Header, err error) { - var ( - bcdBackup *os.File - bcdBackupWriter *winio.BackupFileWriter - ) - if backupPath, ok := mutatedFiles[hdr.Name]; ok { - bcdBackup, err = os.Create(filepath.Join(root, backupPath)) - if err != nil { - return nil, err - } - defer func() { - cerr := bcdBackup.Close() - if err == nil { - err = cerr - } - }() - - bcdBackupWriter = winio.NewBackupFileWriter(bcdBackup, false) - defer func() { - cerr := bcdBackupWriter.Close() - if err == nil { - err = cerr - } - }() - - buf.Reset(io.MultiWriter(w, bcdBackupWriter)) - } else { - buf.Reset(w) - } - - defer func() { - ferr := buf.Flush() - if err == nil { - err = ferr - } - }() - - return backuptar.WriteBackupStreamFromTarFile(buf, t, hdr) -} diff --git a/diff/walking/differ.go b/diff/walking/differ.go index 5ce35910c..7fc2691a1 100644 --- a/diff/walking/differ.go +++ b/diff/walking/differ.go @@ -99,8 +99,8 @@ func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, o if err != nil { cw.Close() if newReference { - if err := s.store.Abort(ctx, config.Reference); err != nil { - log.G(ctx).WithField("ref", config.Reference).Warnf("failed to delete diff upload") + if abortErr := s.store.Abort(ctx, config.Reference); abortErr != nil { + log.G(ctx).WithError(abortErr).WithField("ref", config.Reference).Warnf("failed to delete diff upload") } } } diff --git a/diff/windows/windows.go b/diff/windows/windows.go index 16ecb5bff..7285d2945 100644 --- a/diff/windows/windows.go +++ b/diff/windows/windows.go @@ -20,12 +20,16 @@ package windows import ( "context" + "crypto/rand" + "encoding/base64" + "fmt" "io" "io/ioutil" "time" winio "github.com/Microsoft/go-winio" "github.com/containerd/containerd/archive" + "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" @@ -73,6 +77,7 @@ type windowsDiff struct { } var emptyDesc = ocispec.Descriptor{} +var uncompressed = "containerd.io/uncompressed" // NewWindowsDiff is the Windows container layer implementation // for comparing and applying filesystem layers @@ -94,7 +99,7 @@ func (s windowsDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts "digest": desc.Digest, "size": desc.Size, "media": desc.MediaType, - }).Debugf("diff applied") + }).Debug("diff applied") } }() @@ -135,6 +140,7 @@ func (s windowsDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts // TODO darrenstahlmsft: When this is done isolated, we should disable these. // it currently cannot be disabled, unless we add ref counting. Since this is // temporary, leaving it enabled is OK for now. + // https://github.com/containerd/containerd/issues/1681 if err := winio.EnableProcessPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}); err != nil { return emptyDesc, err } @@ -158,7 +164,136 @@ func (s windowsDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts // Compare creates a diff between the given mounts and uploads the result // to the content store. func (s windowsDiff) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) { - return emptyDesc, errors.Wrap(errdefs.ErrNotImplemented, "windowsDiff does not implement Compare method") + t1 := time.Now() + + var config diff.Config + for _, opt := range opts { + if err := opt(&config); err != nil { + return emptyDesc, err + } + } + + layers, err := mountPairToLayerStack(lower, upper) + if err != nil { + return emptyDesc, err + } + + if config.MediaType == "" { + config.MediaType = ocispec.MediaTypeImageLayerGzip + } + + var isCompressed bool + switch config.MediaType { + case ocispec.MediaTypeImageLayer: + case ocispec.MediaTypeImageLayerGzip: + isCompressed = true + default: + return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType) + } + + newReference := false + if config.Reference == "" { + newReference = true + config.Reference = uniqueRef() + } + + cw, err := s.store.Writer(ctx, content.WithRef(config.Reference), content.WithDescriptor(ocispec.Descriptor{ + MediaType: config.MediaType, + })) + + if err != nil { + return emptyDesc, errors.Wrap(err, "failed to open writer") + } + + defer func() { + if err != nil { + cw.Close() + if newReference { + if abortErr := s.store.Abort(ctx, config.Reference); abortErr != nil { + log.G(ctx).WithError(abortErr).WithField("ref", config.Reference).Warnf("failed to delete diff upload") + } + } + } + }() + + if !newReference { + if err = cw.Truncate(0); err != nil { + return emptyDesc, err + } + } + + // TODO darrenstahlmsft: When this is done isolated, we should disable this. + // it currently cannot be disabled, unless we add ref counting. Since this is + // temporary, leaving it enabled is OK for now. + // https://github.com/containerd/containerd/issues/1681 + if err := winio.EnableProcessPrivileges([]string{winio.SeBackupPrivilege}); err != nil { + return emptyDesc, err + } + + if isCompressed { + dgstr := digest.SHA256.Digester() + var compressed io.WriteCloser + compressed, err = compression.CompressStream(cw, compression.Gzip) + if err != nil { + return emptyDesc, errors.Wrap(err, "failed to get compressed stream") + } + err = archive.WriteDiff(ctx, io.MultiWriter(compressed, dgstr.Hash()), "", layers[0], archive.AsWindowsContainerLayerPair(), archive.WithParentLayers(layers[1:])) + compressed.Close() + if err != nil { + return emptyDesc, errors.Wrap(err, "failed to write compressed diff") + } + + if config.Labels == nil { + config.Labels = map[string]string{} + } + config.Labels[uncompressed] = dgstr.Digest().String() + } else { + if err = archive.WriteDiff(ctx, cw, "", layers[0], archive.AsWindowsContainerLayerPair(), archive.WithParentLayers(layers[1:])); err != nil { + return emptyDesc, errors.Wrap(err, "failed to write diff") + } + } + + var commitopts []content.Opt + if config.Labels != nil { + commitopts = append(commitopts, content.WithLabels(config.Labels)) + } + + dgst := cw.Digest() + if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil { + if !errdefs.IsAlreadyExists(err) { + return emptyDesc, errors.Wrap(err, "failed to commit") + } + } + + info, err := s.store.Info(ctx, dgst) + if err != nil { + return emptyDesc, errors.Wrap(err, "failed to get info from content store") + } + if info.Labels == nil { + info.Labels = make(map[string]string) + } + // Set uncompressed label if digest already existed without label + if _, ok := info.Labels[uncompressed]; !ok { + info.Labels[uncompressed] = config.Labels[uncompressed] + if _, err := s.store.Update(ctx, info, "labels."+uncompressed); err != nil { + return emptyDesc, errors.Wrap(err, "error setting uncompressed label") + } + } + + desc := ocispec.Descriptor{ + MediaType: config.MediaType, + Size: info.Size, + Digest: info.Digest, + } + + log.G(ctx).WithFields(logrus.Fields{ + "d": time.Since(t1), + "dgst": desc.Digest, + "size": desc.Size, + "media": desc.MediaType, + }).Debug("diff created") + + return desc, nil } type readCounter struct { @@ -191,3 +326,58 @@ func mountsToLayerAndParents(mounts []mount.Mount) (string, []string, error) { return mnt.Source, parentLayerPaths, nil } + +// mountPairToLayerStack ensures that the two sets of mount-lists are actually a correct +// parent-and-child, or orphan-and-empty-list, and return the full list of layers, starting +// with the upper-most (most childish?) +func mountPairToLayerStack(lower, upper []mount.Mount) ([]string, error) { + + // May return an ErrNotImplemented, which will fall back to LCOW + upperLayer, upperParentLayerPaths, err := mountsToLayerAndParents(upper) + if err != nil { + return nil, errors.Wrapf(err, "Upper mount invalid") + } + + // Trivial case, diff-against-nothing + if len(lower) == 0 { + if len(upperParentLayerPaths) != 0 { + return nil, errors.Wrap(errdefs.ErrInvalidArgument, "windowsDiff cannot diff a layer with parents against a null layer") + } + return []string{upperLayer}, nil + } + + if len(upperParentLayerPaths) < 1 { + return nil, errors.Wrap(errdefs.ErrInvalidArgument, "windowsDiff cannot diff a layer with no parents against another layer") + } + + lowerLayer, lowerParentLayerPaths, err := mountsToLayerAndParents(lower) + if errdefs.IsNotImplemented(err) { + // Upper was a windows-layer, lower is not. We can't handle that. + return nil, errors.Wrap(errdefs.ErrInvalidArgument, "windowsDiff cannot diff a windows-layer against a non-windows-layer") + } else if err != nil { + return nil, errors.Wrapf(err, "Lower mount invalid") + } + + if upperParentLayerPaths[0] != lowerLayer { + return nil, errors.Wrap(errdefs.ErrInvalidArgument, "windowsDiff cannot diff a layer against a layer other than its own parent") + } + + if len(upperParentLayerPaths) != len(lowerParentLayerPaths)+1 { + return nil, errors.Wrap(errdefs.ErrInvalidArgument, "windowsDiff cannot diff a layer against a layer with different parents") + } + for i, upperParent := range upperParentLayerPaths[1:] { + if upperParent != lowerParentLayerPaths[i] { + return nil, errors.Wrap(errdefs.ErrInvalidArgument, "windowsDiff cannot diff a layer against a layer with different parents") + } + } + + return append([]string{upperLayer}, upperParentLayerPaths...), nil +} + +func uniqueRef() string { + t := time.Now() + var b [3]byte + // Ignore read failures, just decreases uniqueness + rand.Read(b[:]) + return fmt.Sprintf("%d-%s", t.UnixNano(), base64.URLEncoding.EncodeToString(b[:])) +} diff --git a/integration/client/export_test.go b/integration/client/export_test.go index a2c97374e..1e661b99b 100644 --- a/integration/client/export_test.go +++ b/integration/client/export_test.go @@ -20,7 +20,6 @@ import ( "archive/tar" "bytes" "io" - "runtime" "testing" . "github.com/containerd/containerd" @@ -30,8 +29,7 @@ import ( // TestExport exports testImage as a tar stream func TestExport(t *testing.T) { - // TODO: support windows - if testing.Short() || runtime.GOOS == "windows" { + if testing.Short() { t.Skip() } ctx, cancel := testContext(t) diff --git a/integration/client/import_test.go b/integration/client/import_test.go index 6a1415164..22d5603a7 100644 --- a/integration/client/import_test.go +++ b/integration/client/import_test.go @@ -25,7 +25,6 @@ import ( "io/ioutil" "math/rand" "reflect" - "runtime" "testing" . "github.com/containerd/containerd" @@ -41,8 +40,7 @@ import ( // TestExportAndImport exports testImage as a tar stream, // and import the tar stream as a new image. func TestExportAndImport(t *testing.T) { - // TODO: support windows - if testing.Short() || runtime.GOOS == "windows" { + if testing.Short() { t.Skip() } ctx, cancel := testContext(t) diff --git a/vendor/github.com/Microsoft/hcsshim/pkg/ociwclayer/export.go b/vendor/github.com/Microsoft/hcsshim/pkg/ociwclayer/export.go new file mode 100644 index 000000000..d4d800384 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/pkg/ociwclayer/export.go @@ -0,0 +1,86 @@ +// Package ociwclayer provides functions for importing and exporting Windows +// container layers from and to their OCI tar representation. +package ociwclayer + +import ( + "archive/tar" + "context" + "io" + "path/filepath" + + "github.com/Microsoft/go-winio/backuptar" + "github.com/Microsoft/hcsshim" +) + +var driverInfo = hcsshim.DriverInfo{} + +// ExportLayerToTar writes an OCI layer tar stream from the provided on-disk layer. +// The caller must specify the parent layers, if any, ordered from lowest to +// highest layer. +// +// The layer will be mounted for this process, so the caller should ensure that +// it is not currently mounted. +func ExportLayerToTar(ctx context.Context, w io.Writer, path string, parentLayerPaths []string) error { + err := hcsshim.ActivateLayer(driverInfo, path) + if err != nil { + return err + } + defer hcsshim.DeactivateLayer(driverInfo, path) + + // Prepare and unprepare the layer to ensure that it has been initialized. + err = hcsshim.PrepareLayer(driverInfo, path, parentLayerPaths) + if err != nil { + return err + } + err = hcsshim.UnprepareLayer(driverInfo, path) + if err != nil { + return err + } + + r, err := hcsshim.NewLayerReader(driverInfo, path, parentLayerPaths) + if err != nil { + return err + } + + err = writeTarFromLayer(ctx, r, w) + cerr := r.Close() + if err != nil { + return err + } + return cerr +} + +func writeTarFromLayer(ctx context.Context, r hcsshim.LayerReader, w io.Writer) error { + t := tar.NewWriter(w) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + name, size, fileInfo, err := r.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if fileInfo == nil { + // Write a whiteout file. + hdr := &tar.Header{ + Name: filepath.ToSlash(filepath.Join(filepath.Dir(name), whiteoutPrefix+filepath.Base(name))), + } + err := t.WriteHeader(hdr) + if err != nil { + return err + } + } else { + err = backuptar.WriteTarFileFromBackupStream(t, r, name, size, fileInfo) + if err != nil { + return err + } + } + } + return t.Close() +} diff --git a/vendor/github.com/Microsoft/hcsshim/pkg/ociwclayer/import.go b/vendor/github.com/Microsoft/hcsshim/pkg/ociwclayer/import.go new file mode 100644 index 000000000..e74a6b594 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/pkg/ociwclayer/import.go @@ -0,0 +1,148 @@ +package ociwclayer + +import ( + "archive/tar" + "bufio" + "context" + "io" + "os" + "path" + "path/filepath" + "strings" + + winio "github.com/Microsoft/go-winio" + "github.com/Microsoft/go-winio/backuptar" + "github.com/Microsoft/hcsshim" +) + +const whiteoutPrefix = ".wh." + +var ( + // mutatedFiles is a list of files that are mutated by the import process + // and must be backed up and restored. + mutatedFiles = map[string]string{ + "UtilityVM/Files/EFI/Microsoft/Boot/BCD": "bcd.bak", + "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG": "bcd.log.bak", + "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG1": "bcd.log1.bak", + "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG2": "bcd.log2.bak", + } +) + +// ImportLayerFromTar reads a layer from an OCI layer tar stream and extracts it to the +// specified path. The caller must specify the parent layers, if any, ordered +// from lowest to highest layer. +// +// The caller must ensure that the thread or process has acquired backup and +// restore privileges. +// +// This function returns the total size of the layer's files, in bytes. +func ImportLayerFromTar(ctx context.Context, r io.Reader, path string, parentLayerPaths []string) (int64, error) { + err := os.MkdirAll(path, 0) + if err != nil { + return 0, err + } + w, err := hcsshim.NewLayerWriter(hcsshim.DriverInfo{}, path, parentLayerPaths) + if err != nil { + return 0, err + } + n, err := writeLayerFromTar(ctx, r, w, path) + cerr := w.Close() + if err != nil { + return 0, err + } + if cerr != nil { + return 0, cerr + } + return n, nil +} + +func writeLayerFromTar(ctx context.Context, r io.Reader, w hcsshim.LayerWriter, root string) (int64, error) { + t := tar.NewReader(r) + hdr, err := t.Next() + totalSize := int64(0) + buf := bufio.NewWriter(nil) + for err == nil { + select { + case <-ctx.Done(): + return 0, ctx.Err() + default: + } + + base := path.Base(hdr.Name) + if strings.HasPrefix(base, whiteoutPrefix) { + name := path.Join(path.Dir(hdr.Name), base[len(whiteoutPrefix):]) + err = w.Remove(filepath.FromSlash(name)) + if err != nil { + return 0, err + } + hdr, err = t.Next() + } else if hdr.Typeflag == tar.TypeLink { + err = w.AddLink(filepath.FromSlash(hdr.Name), filepath.FromSlash(hdr.Linkname)) + if err != nil { + return 0, err + } + hdr, err = t.Next() + } else { + var ( + name string + size int64 + fileInfo *winio.FileBasicInfo + ) + name, size, fileInfo, err = backuptar.FileInfoFromHeader(hdr) + if err != nil { + return 0, err + } + err = w.Add(filepath.FromSlash(name), fileInfo) + if err != nil { + return 0, err + } + hdr, err = writeBackupStreamFromTarAndSaveMutatedFiles(buf, w, t, hdr, root) + totalSize += size + } + } + if err != io.EOF { + return 0, err + } + return totalSize, nil +} + +// writeBackupStreamFromTarAndSaveMutatedFiles reads data from a tar stream and +// writes it to a backup stream, and also saves any files that will be mutated +// by the import layer process to a backup location. +func writeBackupStreamFromTarAndSaveMutatedFiles(buf *bufio.Writer, w io.Writer, t *tar.Reader, hdr *tar.Header, root string) (nextHdr *tar.Header, err error) { + var bcdBackup *os.File + var bcdBackupWriter *winio.BackupFileWriter + if backupPath, ok := mutatedFiles[hdr.Name]; ok { + bcdBackup, err = os.Create(filepath.Join(root, backupPath)) + if err != nil { + return nil, err + } + defer func() { + cerr := bcdBackup.Close() + if err == nil { + err = cerr + } + }() + + bcdBackupWriter = winio.NewBackupFileWriter(bcdBackup, false) + defer func() { + cerr := bcdBackupWriter.Close() + if err == nil { + err = cerr + } + }() + + buf.Reset(io.MultiWriter(w, bcdBackupWriter)) + } else { + buf.Reset(w) + } + + defer func() { + ferr := buf.Flush() + if err == nil { + err = ferr + } + }() + + return backuptar.WriteBackupStreamFromTarFile(buf, t, hdr) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 29875ecd1..2040087be 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -40,6 +40,7 @@ github.com/Microsoft/hcsshim/internal/wclayer github.com/Microsoft/hcsshim/internal/winapi github.com/Microsoft/hcsshim/osversion github.com/Microsoft/hcsshim/pkg/go-runhcs +github.com/Microsoft/hcsshim/pkg/ociwclayer # github.com/beorn7/perks v1.0.1 github.com/beorn7/perks/quantile # github.com/cespare/xxhash/v2 v2.1.1