Add flat GC label for leases

Provide a flag which configures a lease to only hold
reference to its given references and ignore label references
during garbage collection rooted from the lease.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

gc/gc.go

@@ -30,6 +30,11 @@ import (
 // ResourceType represents type of resource at a node
 type ResourceType uint8
 
+// ResourceMax represents the max resource.
+// Upper bits are stripped out during the mark phase, allowing the upper 3 bits
+// to be used by the caller reference function.
+const ResourceMax = ResourceType(0x1F)
+
 // Node presents a resource which has a type and key,
 // this node can be used to lookup other nodes.
 type Node struct {
@@ -80,6 +85,8 @@ func Tricolor(roots []Node, refs func(ref Node) ([]Node, error)) (map[Node]struc
 			}
 		}
 
+		// strip bits above max resource type
+		id.Type = id.Type & ResourceMax
 		// mark as black when done
 		reachable[id] = struct{}{}
 	}

metadata/db_test.go

@@ -36,6 +36,7 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/gc"
 	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/leases"
 	"github.com/containerd/containerd/log/logtest"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/snapshots"
@@ -345,6 +346,40 @@ func TestMetadataCollector(t *testing.T) {
 			newSnapshot("5", "3", false, true),
 			container("1", "4"),
 			image("image-1", digestFor(2)),
+
+			// Test lease preservation
+			blob(bytesFor(5), false, "containerd.io/gc.ref.content.0", digestFor(6).String()),
+			blob(bytesFor(6), false),
+			blob(bytesFor(7), false),
+			newSnapshot("6", "", false, false, "containerd.io/gc.ref.content.0", digestFor(7).String()),
+			lease("lease-1", []leases.Resource{
+				{
+					ID:   digestFor(5).String(),
+					Type: "content",
+				},
+				{
+					ID:   "6",
+					Type: "snapshots/native",
+				},
+			}, false),
+
+			// Test flat lease
+			blob(bytesFor(8), false, "containerd.io/gc.ref.content.0", digestFor(9).String()),
+			blob(bytesFor(9), true),
+			blob(bytesFor(10), true),
+			newSnapshot("7", "", false, false, "containerd.io/gc.ref.content.0", digestFor(10).String()),
+			newSnapshot("8", "7", false, false),
+			newSnapshot("9", "8", false, false),
+			lease("lease-2", []leases.Resource{
+				{
+					ID:   digestFor(8).String(),
+					Type: "content",
+				},
+				{
+					ID:   "9",
+					Type: "snapshots/native",
+				},
+			}, false, "containerd.io/gc.flat", time.Now().String()),
 		}
 		remaining []gc.Node
 	)
@@ -588,6 +623,26 @@ func create(obj object, tx *bolt.Tx, is images.Store, cs content.Store, sn snaps
 		if err != nil {
 			return nil, err
 		}
+	case testLease:
+		lm := NewLeaseManager(tx)
+		l, err := lm.Create(ctx, leases.WithID(v.id), leases.WithLabels(obj.labels))
+		if err != nil {
+			return nil, err
+		}
+
+		for _, ref := range v.refs {
+			if err := lm.AddResource(ctx, l, ref); err != nil {
+				return nil, err
+			}
+		}
+
+		if !obj.removed {
+			node = &gc.Node{
+				Type:      ResourceLease,
+				Namespace: namespace,
+				Key:       v.id,
+			}
+		}
 	}
 
 	return node, nil
@@ -641,6 +696,17 @@ func container(id, s string, l ...string) object {
 	}
 }
 
+func lease(id string, refs []leases.Resource, r bool, l ...string) object {
+	return object{
+		data: testLease{
+			id:   id,
+			refs: refs,
+		},
+		removed: r,
+		labels:  labelmap(l...),
+	}
+}
+
 type testContent struct {
 	data []byte
 }
@@ -661,6 +727,11 @@ type testContainer struct {
 	snapshot string
 }
 
+type testLease struct {
+	id   string
+	refs []leases.Resource
+}
+
 func newStores(t testing.TB) (*DB, content.Store, snapshots.Snapshotter, func()) {
 	td, err := ioutil.TempDir("", "gc-test-")
 	if err != nil {

metadata/gc.go

@@ -46,11 +46,17 @@ const (
 	ResourceIngest
 )
 
+const (
+	resourceContentFlat  = ResourceContent | 0x20
+	resourceSnapshotFlat = ResourceSnapshot | 0x20
+)
+
 var (
 	labelGCRoot       = []byte("containerd.io/gc.root")
 	labelGCSnapRef    = []byte("containerd.io/gc.ref.snapshot.")
 	labelGCContentRef = []byte("containerd.io/gc.ref.content")
 	labelGCExpire     = []byte("containerd.io/gc.expire")
+	labelGCFlat       = []byte("containerd.io/gc.flat")
 )
 
 func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
@@ -90,6 +96,7 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
 					return nil
 				}
 				libkt := lbkt.Bucket(k)
+				var flat bool
 
 				if lblbkt := libkt.Bucket(bucketKeyObjectLabels); lblbkt != nil {
 					if expV := lblbkt.Get(labelGCExpire); expV != nil {
@@ -102,6 +109,10 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
 							return nil
 						}
 					}
+
+					if flatV := lblbkt.Get(labelGCFlat); flatV != nil {
+						flat = true
+					}
 				}
 
 				fn(gcnode(ResourceLease, ns, string(k)))
@@ -111,16 +122,26 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
 				// no need to allow the lookup to be recursive, handling here
 				// therefore reduces the number of database seeks.
 
+				ctype := ResourceContent
+				if flat {
+					ctype = resourceContentFlat
+				}
+
 				cbkt := libkt.Bucket(bucketKeyObjectContent)
 				if cbkt != nil {
 					if err := cbkt.ForEach(func(k, v []byte) error {
-						fn(gcnode(ResourceContent, ns, string(k)))
+						fn(gcnode(ctype, ns, string(k)))
 						return nil
 					}); err != nil {
 						return err
 					}
 				}
 
+				stype := ResourceSnapshot
+				if flat {
+					stype = resourceSnapshotFlat
+				}
+
 				sbkt := libkt.Bucket(bucketKeyObjectSnapshots)
 				if sbkt != nil {
 					if err := sbkt.ForEach(func(sk, sv []byte) error {
@@ -130,7 +151,7 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
 						snbkt := sbkt.Bucket(sk)
 
 						return snbkt.ForEach(func(k, v []byte) error {
-							fn(gcnode(ResourceSnapshot, ns, fmt.Sprintf("%s/%s", sk, k)))
+							fn(gcnode(stype, ns, fmt.Sprintf("%s/%s", sk, k)))
 							return nil
 						})
 					}); err != nil {
@@ -257,7 +278,8 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
 }
 
 func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node)) error {
-	if node.Type == ResourceContent {
+	switch node.Type {
+	case ResourceContent:
 		bkt := getBucket(tx, bucketKeyVersion, []byte(node.Namespace), bucketKeyObjectContent, bucketKeyObjectBlob, []byte(node.Key))
 		if bkt == nil {
 			// Node may be created from dead edge
@@ -265,7 +287,7 @@ func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node)
 		}
 
 		return sendLabelRefs(node.Namespace, bkt, fn)
-	} else if node.Type == ResourceSnapshot {
+	case ResourceSnapshot, resourceSnapshotFlat:
 		parts := strings.SplitN(node.Key, "/", 2)
 		if len(parts) != 2 {
 			return errors.Errorf("invalid snapshot gc key %s", node.Key)
@@ -280,11 +302,16 @@ func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node)
 		}
 
 		if pv := bkt.Get(bucketKeyParent); len(pv) > 0 {
-			fn(gcnode(ResourceSnapshot, node.Namespace, fmt.Sprintf("%s/%s", ss, pv)))
+			fn(gcnode(node.Type, node.Namespace, fmt.Sprintf("%s/%s", ss, pv)))
+		}
+
+		// Do not send labeled references for flat snapshot refs
+		if node.Type == resourceSnapshotFlat {
+			return nil
 		}
 
 		return sendLabelRefs(node.Namespace, bkt, fn)
-	} else if node.Type == ResourceIngest {
+	case ResourceIngest:
 		// Send expected value
 		bkt := getBucket(tx, bucketKeyVersion, []byte(node.Namespace), bucketKeyObjectContent, bucketKeyObjectIngests, []byte(node.Key))
 		if bkt == nil {

metadata/gc_test.go

@@ -33,6 +33,15 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
+func TestResourceMax(t *testing.T) {
+	if ResourceContent != resourceContentFlat&gc.ResourceMax {
+		t.Fatalf("Invalid flat content type: %d (max %d)", resourceContentFlat, gc.ResourceMax)
+	}
+	if ResourceSnapshot != resourceSnapshotFlat&gc.ResourceMax {
+		t.Fatalf("Invalid flat snapshot type: %d (max %d)", resourceSnapshotFlat, gc.ResourceMax)
+	}
+}
+
 func TestGCRoots(t *testing.T) {
 	db, cleanup, err := newDatabase()
 	if err != nil {
@@ -90,6 +99,11 @@ func TestGCRoots(t *testing.T) {
 		addLeaseSnapshot("ns2", "l4", "overlay", "sn8"),
 		addLeaseIngest("ns2", "l4", "ingest-6"),
 		addLeaseIngest("ns2", "l4", "ingest-7"),
+
+		addLease("ns3", "l1", labelmap(string(labelGCFlat), time.Now().Add(time.Hour).Format(time.RFC3339))),
+		addLeaseContent("ns3", "l1", dgst(1)),
+		addLeaseSnapshot("ns3", "l1", "overlay", "sn1"),
+		addLeaseIngest("ns3", "l1", "ingest-1"),
 	}
 
 	expected := []gc.Node{
@@ -121,6 +135,10 @@ func TestGCRoots(t *testing.T) {
 		gcnode(ResourceIngest, "ns1", "ingest-3"),
 		gcnode(ResourceIngest, "ns2", "ingest-4"),
 		gcnode(ResourceIngest, "ns2", "ingest-5"),
+		gcnode(ResourceLease, "ns3", "l1"),
+		gcnode(ResourceIngest, "ns3", "ingest-1"),
+		gcnode(resourceContentFlat, "ns3", dgst(1).String()),
+		gcnode(resourceSnapshotFlat, "ns3", "overlay/sn1"),
 	}
 
 	if err := db.Update(func(tx *bolt.Tx) error {
@@ -268,6 +286,14 @@ func TestGCRefs(t *testing.T) {
 		addSnapshot("ns2", "overlay", "sn3", "", labelmap(
 			string(labelGCContentRef), dgst(1).String(),
 			string(labelGCContentRef)+".keep-me", dgst(6).String())),
+
+		// Test flat references don't follow label references
+		addContent("ns3", dgst(1), nil),
+		addContent("ns3", dgst(2), labelmap(string(labelGCContentRef)+".0", dgst(1).String())),
+
+		addSnapshot("ns3", "overlay", "sn1", "", nil),
+		addSnapshot("ns3", "overlay", "sn2", "sn1", nil),
+		addSnapshot("ns3", "overlay", "sn3", "", labelmap(string(labelGCSnapRef)+"btrfs", "sn1", string(labelGCSnapRef)+"overlay", "sn1")),
 	}
 
 	refs := map[gc.Node][]gc.Node{
@@ -316,6 +342,18 @@ func TestGCRefs(t *testing.T) {
 		gcnode(ResourceIngest, "ns2", "ingest-2"): {
 			gcnode(ResourceContent, "ns2", dgst(8).String()),
 		},
+		gcnode(resourceSnapshotFlat, "ns3", "overlay/sn2"): {
+			gcnode(resourceSnapshotFlat, "ns3", "overlay/sn1"),
+		},
+		gcnode(ResourceSnapshot, "ns3", "overlay/sn2"): {
+			gcnode(ResourceSnapshot, "ns3", "overlay/sn1"),
+		},
+		gcnode(resourceSnapshotFlat, "ns3", "overlay/sn1"): nil,
+		gcnode(resourceSnapshotFlat, "ns3", "overlay/sn3"): nil,
+		gcnode(ResourceSnapshot, "ns3", "overlay/sn3"): {
+			gcnode(ResourceSnapshot, "ns3", "btrfs/sn1"),
+			gcnode(ResourceSnapshot, "ns3", "overlay/sn1"),
+		},
 	}
 
 	if err := db.Update(func(tx *bolt.Tx) error {