Move diff to core/diff

Signed-off-by: Derek McGowan <derek@mcg.dev>

core/diff/apply/apply.go

@@ -0,0 +1,131 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package apply
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/diff"
+	"github.com/containerd/containerd/v2/mount"
+	"github.com/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// NewFileSystemApplier returns an applier which simply mounts
+// and applies diff onto the mounted filesystem.
+func NewFileSystemApplier(cs content.Provider) diff.Applier {
+	return &fsApplier{
+		store: cs,
+	}
+}
+
+type fsApplier struct {
+	store content.Provider
+}
+
+var emptyDesc = ocispec.Descriptor{}
+
+// Apply applies the content associated with the provided digests onto the
+// provided mounts. Archive content will be extracted and decompressed if
+// necessary.
+func (s *fsApplier) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispec.Descriptor, err error) {
+	t1 := time.Now()
+	defer func() {
+		if err == nil {
+			log.G(ctx).WithFields(log.Fields{
+				"d":      time.Since(t1),
+				"digest": desc.Digest,
+				"size":   desc.Size,
+				"media":  desc.MediaType,
+			}).Debugf("diff applied")
+		}
+	}()
+
+	var config diff.ApplyConfig
+	for _, o := range opts {
+		if err := o(ctx, desc, &config); err != nil {
+			return emptyDesc, fmt.Errorf("failed to apply config opt: %w", err)
+		}
+	}
+
+	ra, err := s.store.ReaderAt(ctx, desc)
+	if err != nil {
+		return emptyDesc, fmt.Errorf("failed to get reader from content store: %w", err)
+	}
+	defer ra.Close()
+
+	var processors []diff.StreamProcessor
+	processor := diff.NewProcessorChain(desc.MediaType, content.NewReader(ra))
+	processors = append(processors, processor)
+	for {
+		if processor, err = diff.GetProcessor(ctx, processor, config.ProcessorPayloads); err != nil {
+			return emptyDesc, fmt.Errorf("failed to get stream processor for %s: %w", desc.MediaType, err)
+		}
+		processors = append(processors, processor)
+		if processor.MediaType() == ocispec.MediaTypeImageLayer {
+			break
+		}
+	}
+	defer processor.Close()
+
+	digester := digest.Canonical.Digester()
+	rc := &readCounter{
+		r: io.TeeReader(processor, digester.Hash()),
+	}
+
+	if err := apply(ctx, mounts, rc, config.SyncFs); err != nil {
+		return emptyDesc, err
+	}
+
+	// Read any trailing data
+	if _, err := io.Copy(io.Discard, rc); err != nil {
+		return emptyDesc, err
+	}
+
+	for _, p := range processors {
+		if ep, ok := p.(interface {
+			Err() error
+		}); ok {
+			if err := ep.Err(); err != nil {
+				return emptyDesc, err
+			}
+		}
+	}
+	return ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageLayer,
+		Size:      rc.c,
+		Digest:    digester.Digest(),
+	}, nil
+}
+
+type readCounter struct {
+	r io.Reader
+	c int64
+}
+
+func (rc *readCounter) Read(p []byte) (n int, err error) {
+	n, err = rc.r.Read(p)
+	if n > 0 {
+		rc.c += int64(n)
+	}
+	return
+}

core/diff/apply/apply_darwin.go

@@ -0,0 +1,49 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package apply
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/containerd/containerd/v2/archive"
+	"github.com/containerd/containerd/v2/mount"
+)
+
+func apply(ctx context.Context, mounts []mount.Mount, r io.Reader, _sync bool) error {
+	// We currently do not support mounts nor bind mounts on MacOS in the containerd daemon.
+	// Using this as an exception to enable native snapshotter and allow further research.
+	if len(mounts) == 1 && mounts[0].Type == "bind" {
+		opts := []archive.ApplyOpt{}
+
+		if os.Getuid() != 0 {
+			opts = append(opts, archive.WithNoSameOwner())
+		}
+
+		path := mounts[0].Source
+		_, err := archive.Apply(ctx, path, r, opts...)
+		return err
+
+		// TODO: Do we need to sync all the filesystems?
+	}
+
+	return mount.WithTempMount(ctx, mounts, func(root string) error {
+		_, err := archive.Apply(ctx, root, r)
+		return err
+	})
+}

core/diff/apply/apply_linux.go

@@ -0,0 +1,105 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package apply
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/containerd/containerd/v2/archive"
+	"github.com/containerd/containerd/v2/errdefs"
+	"github.com/containerd/containerd/v2/mount"
+	"github.com/containerd/containerd/v2/pkg/userns"
+
+	"golang.org/x/sys/unix"
+)
+
+func apply(ctx context.Context, mounts []mount.Mount, r io.Reader, sync bool) (retErr error) {
+	switch {
+	case len(mounts) == 1 && mounts[0].Type == "overlay":
+		// OverlayConvertWhiteout (mknod c 0 0) doesn't work in userns.
+		// https://github.com/containerd/containerd/issues/3762
+		if userns.RunningInUserNS() {
+			break
+		}
+		path, parents, err := getOverlayPath(mounts[0].Options)
+		if err != nil {
+			if errdefs.IsInvalidArgument(err) {
+				break
+			}
+			return err
+		}
+		opts := []archive.ApplyOpt{
+			archive.WithConvertWhiteout(archive.OverlayConvertWhiteout),
+		}
+		if len(parents) > 0 {
+			opts = append(opts, archive.WithParents(parents))
+		}
+		_, err = archive.Apply(ctx, path, r, opts...)
+		if err == nil && sync {
+			err = doSyncFs(path)
+		}
+		return err
+	case sync && len(mounts) == 1 && mounts[0].Type == "bind":
+		defer func() {
+			if retErr != nil {
+				return
+			}
+
+			retErr = doSyncFs(mounts[0].Source)
+		}()
+	}
+	return mount.WithTempMount(ctx, mounts, func(root string) error {
+		_, err := archive.Apply(ctx, root, r)
+		return err
+	})
+}
+
+func getOverlayPath(options []string) (upper string, lower []string, err error) {
+	const upperdirPrefix = "upperdir="
+	const lowerdirPrefix = "lowerdir="
+
+	for _, o := range options {
+		if strings.HasPrefix(o, upperdirPrefix) {
+			upper = strings.TrimPrefix(o, upperdirPrefix)
+		} else if strings.HasPrefix(o, lowerdirPrefix) {
+			lower = strings.Split(strings.TrimPrefix(o, lowerdirPrefix), ":")
+		}
+	}
+	if upper == "" {
+		return "", nil, fmt.Errorf("upperdir not found: %w", errdefs.ErrInvalidArgument)
+	}
+
+	return
+}
+
+func doSyncFs(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("failed to open %s: %w", file, err)
+	}
+	defer fd.Close()
+
+	_, _, errno := unix.Syscall(unix.SYS_SYNCFS, fd.Fd(), 0, 0)
+	if errno != 0 {
+		return fmt.Errorf("failed to syncfs for %s: %w", file, errno)
+	}
+	return nil
+}

core/diff/apply/apply_linux_test.go

@@ -0,0 +1,41 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package apply
+
+import (
+	"testing"
+)
+
+func TestGetOverlayPath(t *testing.T) {
+	good := []string{"upperdir=/test/upper", "lowerdir=/test/lower1:/test/lower2", "workdir=/test/work"}
+	path, parents, err := getOverlayPath(good)
+	if err != nil {
+		t.Fatalf("Get overlay path failed: %v", err)
+	}
+	if path != "/test/upper" {
+		t.Fatalf("Unexpected upperdir: %q", path)
+	}
+	if len(parents) != 2 || parents[0] != "/test/lower1" || parents[1] != "/test/lower2" {
+		t.Fatalf("Unexpected parents: %v", parents)
+	}
+
+	bad := []string{"lowerdir=/test/lower"}
+	_, _, err = getOverlayPath(bad)
+	if err == nil {
+		t.Fatalf("An error is expected")
+	}
+}

core/diff/apply/apply_other.go

@@ -0,0 +1,35 @@
+//go:build !linux && !darwin
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package apply
+
+import (
+	"context"
+	"io"
+
+	"github.com/containerd/containerd/v2/archive"
+	"github.com/containerd/containerd/v2/mount"
+)
+
+func apply(ctx context.Context, mounts []mount.Mount, r io.Reader, _sync bool) error {
+	// TODO: for windows, how to sync?
+	return mount.WithTempMount(ctx, mounts, func(root string) error {
+		_, err := archive.Apply(ctx, root, r)
+		return err
+	})
+}