archive: add WithSourceDateEpoch() for whiteouts

This makes diff archives to be reproducible.

The value is expected to be passed from CLI applications via the $SOUCE_DATE_EPOCH env var.

See https://reproducible-builds.org/docs/source-date-epoch/
for the $SOURCE_DATE_EPOCH specification.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

archive/tar.go

@@ -50,11 +50,11 @@ var errInvalidArchive = errors.New("invalid archive")
 // files will be prepended with the prefix ".wh.". This style is
 // based off AUFS whiteouts.
 // See https://github.com/opencontainers/image-spec/blob/main/layer.md
-func Diff(ctx context.Context, a, b string) io.ReadCloser {
+func Diff(ctx context.Context, a, b string, opts ...WriteDiffOpt) io.ReadCloser {
 	r, w := io.Pipe()
 
 	go func() {
-		err := WriteDiff(ctx, w, a, b)
+		err := WriteDiff(ctx, w, a, b, opts...)
 		if err != nil {
 			log.G(ctx).WithError(err).Debugf("write diff failed")
 		}
@@ -94,8 +94,12 @@ func WriteDiff(ctx context.Context, w io.Writer, a, b string, opts ...WriteDiffO
 // files will be prepended with the prefix ".wh.". This style is
 // based off AUFS whiteouts.
 // See https://github.com/opencontainers/image-spec/blob/main/layer.md
-func writeDiffNaive(ctx context.Context, w io.Writer, a, b string, _ WriteDiffOptions) error {
-	cw := NewChangeWriter(w, b)
+func writeDiffNaive(ctx context.Context, w io.Writer, a, b string, o WriteDiffOptions) error {
+	var opts []ChangeWriterOpt
+	if o.SourceDateEpoch != nil {
+		opts = append(opts, WithWhiteoutTime(*o.SourceDateEpoch))
+	}
+	cw := NewChangeWriter(w, b, opts...)
 	err := fs.Changes(ctx, a, b, cw.HandleChange)
 	if err != nil {
 		return fmt.Errorf("failed to create diff tar stream: %w", err)
@@ -497,18 +501,32 @@ type ChangeWriter struct {
 	addedDirs map[string]struct{}
 }
 
+// ChangeWriterOpt can be specified in NewChangeWriter.
+type ChangeWriterOpt func(cw *ChangeWriter)
+
+// WithWhiteoutTime sets the whiteout timestamp.
+func WithWhiteoutTime(tm time.Time) ChangeWriterOpt {
+	return func(cw *ChangeWriter) {
+		cw.whiteoutT = tm
+	}
+}
+
 // NewChangeWriter returns ChangeWriter that writes tar stream of the source directory
 // to the privided writer. Change information (add/modify/delete/unmodified) for each
 // file needs to be passed through HandleChange method.
-func NewChangeWriter(w io.Writer, source string) *ChangeWriter {
-	return &ChangeWriter{
+func NewChangeWriter(w io.Writer, source string, opts ...ChangeWriterOpt) *ChangeWriter {
+	cw := &ChangeWriter{
 		tw:        tar.NewWriter(w),
 		source:    source,
-		whiteoutT: time.Now(),
+		whiteoutT: time.Now(), // can be overridden with WithWhiteoutTime(time.Time) ChangeWriterOpt .
 		inodeSrc:  map[uint64]string{},
 		inodeRefs: map[uint64][]string{},
 		addedDirs: map[string]struct{}{},
 	}
+	for _, o := range opts {
+		o(cw)
+	}
+	return cw
 }
 
 // HandleChange receives filesystem change information and reflect that information to

archive/tar_opts.go

@@ -20,6 +20,7 @@ import (
 	"archive/tar"
 	"context"
 	"io"
+	"time"
 )
 
 // ApplyOptions provides additional options for an Apply operation
@@ -89,7 +90,19 @@ type WriteDiffOptions struct {
 	ParentLayers []string // Windows needs the full list of parent layers
 
 	writeDiffFunc func(context.Context, io.Writer, string, string, WriteDiffOptions) error
+
+	// SourceDateEpoch specifies the timestamp used for whiteouts to provide control for reproducibility.
+	// See also https://reproducible-builds.org/docs/source-date-epoch/ .
+	SourceDateEpoch *time.Time
 }
 
 // WriteDiffOpt allows setting mutable archive write properties on creation
 type WriteDiffOpt func(options *WriteDiffOptions) error
+
+// WithSourceDateEpoch specifies the SOURCE_DATE_EPOCH without touching the env vars.
+func WithSourceDateEpoch(tm *time.Time) WriteDiffOpt {
+	return func(options *WriteDiffOptions) error {
+		options.SourceDateEpoch = tm
+		return nil
+	}
+}

archive/tar_test.go

@@ -36,6 +36,7 @@ import (
 	"github.com/containerd/containerd/pkg/testutil"
 	"github.com/containerd/continuity/fs"
 	"github.com/containerd/continuity/fs/fstest"
+	"github.com/stretchr/testify/require"
 	exec "golang.org/x/sys/execabs"
 )
 
@@ -1156,8 +1157,35 @@ func TestDiffTar(t *testing.T) {
 	}
 }
 
+func TestWhiteoutSourceDateEpoch(t *testing.T) {
+	sourceDateEpoch, err := time.Parse(time.RFC3339, "2022-01-23T12:34:56Z")
+	require.NoError(t, err)
+	opts := []WriteDiffOpt{WithSourceDateEpoch(&sourceDateEpoch)}
+	validators := []tarEntryValidator{
+		composeValidators(whiteoutEntry("f1"), requireModTime(sourceDateEpoch)),
+	}
+	a := fstest.Apply(
+		fstest.CreateFile("/f1", []byte("content"), 0644),
+	)
+	b := fstest.Apply(
+		fstest.RemoveAll("/f1"),
+	)
+	makeDiffTarTest(validators, a, b, opts...)(t)
+}
+
 type tarEntryValidator func(*tar.Header, []byte) error
 
+func composeValidators(vv ...tarEntryValidator) tarEntryValidator {
+	return func(hdr *tar.Header, b []byte) error {
+		for _, v := range vv {
+			if err := v(hdr, b); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
 func dirEntry(name string, mode int) tarEntryValidator {
 	return func(hdr *tar.Header, b []byte) error {
 		if hdr.Typeflag != tar.TypeDir {
@@ -1222,7 +1250,16 @@ func whiteoutEntry(name string) tarEntryValidator {
 	}
 }
 
-func makeDiffTarTest(validators []tarEntryValidator, a, b fstest.Applier) func(*testing.T) {
+func requireModTime(expected time.Time) tarEntryValidator {
+	return func(hdr *tar.Header, b []byte) error {
+		if !hdr.ModTime.Equal(expected) {
+			return fmt.Errorf("expected ModTime %v, got %v", expected, hdr.ModTime)
+		}
+		return nil
+	}
+}
+
+func makeDiffTarTest(validators []tarEntryValidator, a, b fstest.Applier, opts ...WriteDiffOpt) func(*testing.T) {
 	return func(t *testing.T) {
 		ad := t.TempDir()
 		if err := a.Apply(ad); err != nil {
@@ -1237,7 +1274,7 @@ func makeDiffTarTest(validators []tarEntryValidator, a, b fstest.Applier) func(*
 			t.Fatalf("failed to apply b: %v", err)
 		}
 
-		rc := Diff(context.Background(), ad, bd)
+		rc := Diff(context.Background(), ad, bd, opts...)
 		defer rc.Close()
 
 		tr := tar.NewReader(rc)