Add snapshot walk implementations

Temporarily remove zfs and aufs until interface update

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

snapshots/btrfs/btrfs.go

@@ -186,13 +186,13 @@ func (b *snapshotter) usage(ctx context.Context, key string) (snapshots.Usage, e
 }
 
 // Walk the committed snapshots.
-func (b *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error, filters ...string) error {
+func (b *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
 	ctx, t, err := b.ms.TransactionContext(ctx, false)
 	if err != nil {
 		return err
 	}
 	defer t.Rollback()
-	return storage.WalkInfo(ctx, fn)
+	return storage.WalkInfo(ctx, fn, fs...)
 }
 
 func (b *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {

snapshots/devmapper/snapshotter.go

@@ -312,10 +312,10 @@ func (s *Snapshotter) removeDevice(ctx context.Context, key string) error {
 }
 
 // Walk iterates through all metadata Info for the stored snapshots and calls the provided function for each.
-func (s *Snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error, filters ...string) error {
+func (s *Snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
 	log.G(ctx).Debug("walk")
 	return s.withTransaction(ctx, false, func(ctx context.Context) error {
-		return storage.WalkInfo(ctx, fn)
+		return storage.WalkInfo(ctx, fn, fs...)
 	})
 }
 

snapshots/lcow/lcow.go

@@ -241,7 +241,7 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
 }
 
 // Walk the committed snapshots.
-func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error, filters ...string) error {
+func (s *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
 	log.G(ctx).Debug("Starting Walk")
 	ctx, t, err := s.ms.TransactionContext(ctx, false)
 	if err != nil {
@@ -249,7 +249,7 @@ func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho
 	}
 	defer t.Rollback()
 
-	return storage.WalkInfo(ctx, fn)
+	return storage.WalkInfo(ctx, fn, fs...)
 }
 
 // Close closes the snapshotter

snapshots/native/native.go

@@ -232,13 +232,13 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) {
 }
 
 // Walk the committed snapshots.
-func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error, filters ...string) error {
+func (o *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
 	ctx, t, err := o.ms.TransactionContext(ctx, false)
 	if err != nil {
 		return err
 	}
 	defer t.Rollback()
-	return storage.WalkInfo(ctx, fn)
+	return storage.WalkInfo(ctx, fn, fs...)
 }
 
 func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ []mount.Mount, err error) {

snapshots/overlay/overlay.go

@@ -282,14 +282,14 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) {
 	return t.Commit()
 }
 
-// Walk the committed snapshots.
-func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error, filters ...string) error {
+// Walk the snapshots.
+func (o *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
 	ctx, t, err := o.ms.TransactionContext(ctx, false)
 	if err != nil {
 		return err
 	}
 	defer t.Rollback()
-	return storage.WalkInfo(ctx, fn)
+	return storage.WalkInfo(ctx, fn, fs...)
 }
 
 // Cleanup cleans up disk resources from removed or abandoned snapshots

snapshots/proxy/proxy.go

@@ -153,9 +153,10 @@ func (p *proxySnapshotter) Remove(ctx context.Context, key string) error {
 	return errdefs.FromGRPC(err)
 }
 
-func (p *proxySnapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error, filters ...string) error {
+func (p *proxySnapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
 	sc, err := p.client.List(ctx, &snapshotsapi.ListSnapshotsRequest{
 		Snapshotter: p.snapshotterName,
+		Filters:     fs,
 	})
 	if err != nil {
 		return errdefs.FromGRPC(err)

snapshots/snapshotter.go

@@ -118,6 +118,9 @@ func (u *Usage) Add(other Usage) {
 	u.Inodes += other.Inodes
 }
 
+// WalkFunc defines the callback for a snapshot walk.
+type WalkFunc func(context.Context, Info) error
+
 // Snapshotter defines the methods required to implement a snapshot snapshotter for
 // allocating, snapshotting and mounting filesystem changesets. The model works
 // by building up sets of changes with parent-child relationships.
@@ -314,12 +317,15 @@ type Snapshotter interface {
 	// removed before proceeding.
 	Remove(ctx context.Context, key string) error
 
-	// Walk all snapshots in the snapshotter. For each snapshot in the
-	// snapshotter, the function will be called, unless filters were used.
-	// Zero or more filters may be provided as strings. Only events that match
-	// *any* of the provided filters will be sent on the channel. The filters use
-	// the standard containerd filters package syntax.
-	Walk(ctx context.Context, fn func(context.Context, Info) error, filters ...string) error
+	// Walk will call the provided function for each snapshot in the
+	// snapshotter which match the provided filters. If no filters are
+	// given all items will be walked.
+	// Filters:
+	//  name
+	//  parent
+	//  kind (active,view,committed)
+	//  labels.(label)
+	Walk(ctx context.Context, fn WalkFunc, filters ...string) error
 
 	// Close releases the internal resources.
 	//

snapshots/storage/bolt.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/filters"
 	"github.com/containerd/containerd/metadata/boltutil"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/pkg/errors"
@@ -144,7 +145,12 @@ func UpdateInfo(ctx context.Context, info snapshots.Info, fieldpaths ...string)
 // WalkInfo iterates through all metadata Info for the stored snapshots and
 // calls the provided function for each. Requires a context with a storage
 // transaction.
-func WalkInfo(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
+func WalkInfo(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
+	filter, err := filters.ParseAll(fs...)
+	if err != nil {
+		return err
+	}
+	// TODO: allow indexes (name, parent, specific labels)
 	return withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
 		return bkt.ForEach(func(k, v []byte) error {
 			// skip non buckets
@@ -160,6 +166,9 @@ func WalkInfo(ctx context.Context, fn func(context.Context, snapshots.Info) erro
 			if err := readSnapshot(sbkt, nil, &si); err != nil {
 				return err
 			}
+			if !filter.Match(adaptSnapshot(si)) {
+				return nil
+			}
 
 			return fn(ctx, si)
 		})
@@ -604,3 +613,38 @@ func encodeID(id uint64) ([]byte, error) {
 	}
 	return idEncoded, nil
 }
+
+func adaptSnapshot(info snapshots.Info) filters.Adaptor {
+	return filters.AdapterFunc(func(fieldpath []string) (string, bool) {
+		if len(fieldpath) == 0 {
+			return "", false
+		}
+
+		switch fieldpath[0] {
+		case "kind":
+			switch info.Kind {
+			case snapshots.KindActive:
+				return "active", true
+			case snapshots.KindView:
+				return "view", true
+			case snapshots.KindCommitted:
+				return "committed", true
+			}
+		case "name":
+			return info.Name, true
+		case "parent":
+			if info.Parent != "" {
+				return info.Parent, true
+			}
+		case "labels":
+			if len(info.Labels) == 0 {
+				return "", false
+			}
+
+			v, ok := info.Labels[strings.Join(fieldpath[1:], ".")]
+			return v, ok
+		}
+
+		return "", false
+	})
+}

snapshots/testsuite/testsuite.go

@@ -23,6 +23,7 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"sort"
 	"testing"
 	"time"
 
@@ -49,6 +50,7 @@ func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.
 	t.Run("PreareViewFailingtest", makeTest(name, snapshotterFn, checkSnapshotterPrepareView))
 	t.Run("Update", makeTest(name, snapshotterFn, checkUpdate))
 	t.Run("Remove", makeTest(name, snapshotterFn, checkRemove))
+	t.Run("Walk", makeTest(name, snapshotterFn, checkWalk))
 
 	t.Run("LayerFileupdate", makeTest(name, snapshotterFn, checkLayerFileUpdate))
 	t.Run("RemoveDirectoryInLowerLayer", makeTest(name, snapshotterFn, checkRemoveDirectoryInLowerLayer))
@@ -961,3 +963,135 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn
 		}
 	}
 }
+
+func checkWalk(ctx context.Context, t *testing.T, snapshotter snapshots.Snapshotter, work string) {
+	opt := snapshots.WithLabels(map[string]string{
+		"containerd.io/gc.root": "check-walk",
+	})
+
+	// No parent active
+	if _, err := snapshotter.Prepare(ctx, "a-np", "", opt); err != nil {
+		t.Fatal(err)
+	}
+
+	// Base parent
+	if _, err := snapshotter.Prepare(ctx, "p-tmp", "", opt); err != nil {
+		t.Fatal(err)
+	}
+	if err := snapshotter.Commit(ctx, "p", "p-tmp", opt); err != nil {
+		t.Fatal(err)
+	}
+
+	// Active
+	if _, err := snapshotter.Prepare(ctx, "a", "p", opt); err != nil {
+		t.Fatal(err)
+	}
+
+	// View
+	if _, err := snapshotter.View(ctx, "v", "p", opt); err != nil {
+		t.Fatal(err)
+	}
+
+	// Base parent with label=1
+	if _, err := snapshotter.Prepare(ctx, "p-wl-tmp", "", opt); err != nil {
+		t.Fatal(err)
+	}
+	if err := snapshotter.Commit(ctx, "p-wl", "p-wl-tmp", snapshots.WithLabels(map[string]string{
+		"l":                     "1",
+		"containerd.io/gc.root": "check-walk",
+	})); err != nil {
+		t.Fatal(err)
+	}
+
+	// active with label=2
+	if _, err := snapshotter.Prepare(ctx, "a-wl", "p-wl", snapshots.WithLabels(map[string]string{
+		"l":                     "2",
+		"containerd.io/gc.root": "check-walk",
+	})); err != nil {
+		t.Fatal(err)
+	}
+
+	// view with label=3
+	if _, err := snapshotter.View(ctx, "v-wl", "p-wl", snapshots.WithLabels(map[string]string{
+		"l":                     "3",
+		"containerd.io/gc.root": "check-walk",
+	})); err != nil {
+		t.Fatal(err)
+	}
+
+	// no parent active with label=2
+	if _, err := snapshotter.Prepare(ctx, "a-np-wl", "", snapshots.WithLabels(map[string]string{
+		"l":                     "2",
+		"containerd.io/gc.root": "check-walk",
+	})); err != nil {
+		t.Fatal(err)
+	}
+
+	for i, tc := range []struct {
+		matches []string
+		filters []string
+	}{
+		{
+			matches: []string{"a-np", "p", "a", "v", "p-wl", "a-wl", "v-wl", "a-np-wl"},
+			filters: []string{"labels.\"containerd.io/gc.root\"==check-walk"},
+		},
+		{
+			matches: []string{"a-np", "a", "a-wl", "a-np-wl"},
+			filters: []string{"kind==active,labels.\"containerd.io/gc.root\"==check-walk"},
+		},
+		{
+			matches: []string{"v", "v-wl"},
+			filters: []string{"kind==view,labels.\"containerd.io/gc.root\"==check-walk"},
+		},
+		{
+			matches: []string{"p", "p-wl"},
+			filters: []string{"kind==committed,labels.\"containerd.io/gc.root\"==check-walk"},
+		},
+		{
+			matches: []string{"p", "a-np-wl"},
+			filters: []string{"name==p", "name==a-np-wl"},
+		},
+		{
+			matches: []string{"a-wl"},
+			filters: []string{"name==a-wl,labels.l"},
+		},
+		{
+			matches: []string{"a", "v"},
+			filters: []string{"parent==p"},
+		},
+		{
+			matches: []string{"a", "v", "a-wl", "v-wl"},
+			filters: []string{"parent,labels.\"containerd.io/gc.root\"==check-walk"},
+		},
+		{
+			matches: []string{"p-wl", "a-wl", "v-wl", "a-np-wl"},
+			filters: []string{"labels.l"},
+		},
+		{
+			matches: []string{"a-wl", "a-np-wl"},
+			filters: []string{"labels.l==2"},
+		},
+	} {
+		actual := []string{}
+		err := snapshotter.Walk(ctx, func(ctx context.Context, si snapshots.Info) error {
+			actual = append(actual, si.Name)
+			return nil
+		}, tc.filters...)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		sort.Strings(tc.matches)
+		sort.Strings(actual)
+		if len(actual) != len(tc.matches) {
+			t.Errorf("[%d] Unexpected result (size):\nActual:\n\t%#v\nExpected:\n\t%#v", i, actual, tc.matches)
+			continue
+		}
+		for j := range actual {
+			if actual[j] != tc.matches[j] {
+				t.Errorf("[%d] Unexpected result @%d:\nActual:\n\t%#vExpected:\n\t%#v", i, j, actual, tc.matches)
+				break
+			}
+		}
+	}
+}

snapshots/windows/windows.go

@@ -238,14 +238,14 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
 }
 
 // Walk the committed snapshots.
-func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error, filters ...string) error {
+func (s *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
 	ctx, t, err := s.ms.TransactionContext(ctx, false)
 	if err != nil {
 		return err
 	}
 	defer t.Rollback()
 
-	return storage.WalkInfo(ctx, fn)
+	return storage.WalkInfo(ctx, fn, fs...)
 }
 
 // Close closes the snapshotter