From dd0a45dfe05d2d3fde956d2396ba31b9b68f94b0 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 14 Jun 2019 10:05:22 +0800 Subject: [PATCH] Add flat GC label for leases Provide a flag which configures a lease to only hold reference to its given references and ignore label references during garbage collection rooted from the lease. Signed-off-by: Derek McGowan --- gc/gc.go | 7 +++++ metadata/db_test.go | 71 +++++++++++++++++++++++++++++++++++++++++++++ metadata/gc.go | 39 +++++++++++++++++++++---- metadata/gc_test.go | 38 ++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 6 deletions(-) diff --git a/gc/gc.go b/gc/gc.go index 35a1712cb..c6fcf7910 100644 --- a/gc/gc.go +++ b/gc/gc.go @@ -30,6 +30,11 @@ import ( // ResourceType represents type of resource at a node type ResourceType uint8 +// ResourceMax represents the max resource. +// Upper bits are stripped out during the mark phase, allowing the upper 3 bits +// to be used by the caller reference function. +const ResourceMax = ResourceType(0x1F) + // Node presents a resource which has a type and key, // this node can be used to lookup other nodes. type Node struct { @@ -80,6 +85,8 @@ func Tricolor(roots []Node, refs func(ref Node) ([]Node, error)) (map[Node]struc } } + // strip bits above max resource type + id.Type = id.Type & ResourceMax // mark as black when done reachable[id] = struct{}{} } diff --git a/metadata/db_test.go b/metadata/db_test.go index 64632db02..a1ae62e69 100644 --- a/metadata/db_test.go +++ b/metadata/db_test.go @@ -36,6 +36,7 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" "github.com/containerd/containerd/log/logtest" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/snapshots" @@ -345,6 +346,40 @@ func TestMetadataCollector(t *testing.T) { newSnapshot("5", "3", false, true), container("1", "4"), image("image-1", digestFor(2)), + + // Test lease preservation + blob(bytesFor(5), false, "containerd.io/gc.ref.content.0", digestFor(6).String()), + blob(bytesFor(6), false), + blob(bytesFor(7), false), + newSnapshot("6", "", false, false, "containerd.io/gc.ref.content.0", digestFor(7).String()), + lease("lease-1", []leases.Resource{ + { + ID: digestFor(5).String(), + Type: "content", + }, + { + ID: "6", + Type: "snapshots/native", + }, + }, false), + + // Test flat lease + blob(bytesFor(8), false, "containerd.io/gc.ref.content.0", digestFor(9).String()), + blob(bytesFor(9), true), + blob(bytesFor(10), true), + newSnapshot("7", "", false, false, "containerd.io/gc.ref.content.0", digestFor(10).String()), + newSnapshot("8", "7", false, false), + newSnapshot("9", "8", false, false), + lease("lease-2", []leases.Resource{ + { + ID: digestFor(8).String(), + Type: "content", + }, + { + ID: "9", + Type: "snapshots/native", + }, + }, false, "containerd.io/gc.flat", time.Now().String()), } remaining []gc.Node ) @@ -588,6 +623,26 @@ func create(obj object, tx *bolt.Tx, is images.Store, cs content.Store, sn snaps if err != nil { return nil, err } + case testLease: + lm := NewLeaseManager(tx) + l, err := lm.Create(ctx, leases.WithID(v.id), leases.WithLabels(obj.labels)) + if err != nil { + return nil, err + } + + for _, ref := range v.refs { + if err := lm.AddResource(ctx, l, ref); err != nil { + return nil, err + } + } + + if !obj.removed { + node = &gc.Node{ + Type: ResourceLease, + Namespace: namespace, + Key: v.id, + } + } } return node, nil @@ -641,6 +696,17 @@ func container(id, s string, l ...string) object { } } +func lease(id string, refs []leases.Resource, r bool, l ...string) object { + return object{ + data: testLease{ + id: id, + refs: refs, + }, + removed: r, + labels: labelmap(l...), + } +} + type testContent struct { data []byte } @@ -661,6 +727,11 @@ type testContainer struct { snapshot string } +type testLease struct { + id string + refs []leases.Resource +} + func newStores(t testing.TB) (*DB, content.Store, snapshots.Snapshotter, func()) { td, err := ioutil.TempDir("", "gc-test-") if err != nil { diff --git a/metadata/gc.go b/metadata/gc.go index 6afaa1772..afe16c922 100644 --- a/metadata/gc.go +++ b/metadata/gc.go @@ -46,11 +46,17 @@ const ( ResourceIngest ) +const ( + resourceContentFlat = ResourceContent | 0x20 + resourceSnapshotFlat = ResourceSnapshot | 0x20 +) + var ( labelGCRoot = []byte("containerd.io/gc.root") labelGCSnapRef = []byte("containerd.io/gc.ref.snapshot.") labelGCContentRef = []byte("containerd.io/gc.ref.content") labelGCExpire = []byte("containerd.io/gc.expire") + labelGCFlat = []byte("containerd.io/gc.flat") ) func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error { @@ -90,6 +96,7 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error { return nil } libkt := lbkt.Bucket(k) + var flat bool if lblbkt := libkt.Bucket(bucketKeyObjectLabels); lblbkt != nil { if expV := lblbkt.Get(labelGCExpire); expV != nil { @@ -102,6 +109,10 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error { return nil } } + + if flatV := lblbkt.Get(labelGCFlat); flatV != nil { + flat = true + } } fn(gcnode(ResourceLease, ns, string(k))) @@ -111,16 +122,26 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error { // no need to allow the lookup to be recursive, handling here // therefore reduces the number of database seeks. + ctype := ResourceContent + if flat { + ctype = resourceContentFlat + } + cbkt := libkt.Bucket(bucketKeyObjectContent) if cbkt != nil { if err := cbkt.ForEach(func(k, v []byte) error { - fn(gcnode(ResourceContent, ns, string(k))) + fn(gcnode(ctype, ns, string(k))) return nil }); err != nil { return err } } + stype := ResourceSnapshot + if flat { + stype = resourceSnapshotFlat + } + sbkt := libkt.Bucket(bucketKeyObjectSnapshots) if sbkt != nil { if err := sbkt.ForEach(func(sk, sv []byte) error { @@ -130,7 +151,7 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error { snbkt := sbkt.Bucket(sk) return snbkt.ForEach(func(k, v []byte) error { - fn(gcnode(ResourceSnapshot, ns, fmt.Sprintf("%s/%s", sk, k))) + fn(gcnode(stype, ns, fmt.Sprintf("%s/%s", sk, k))) return nil }) }); err != nil { @@ -257,7 +278,8 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error { } func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node)) error { - if node.Type == ResourceContent { + switch node.Type { + case ResourceContent: bkt := getBucket(tx, bucketKeyVersion, []byte(node.Namespace), bucketKeyObjectContent, bucketKeyObjectBlob, []byte(node.Key)) if bkt == nil { // Node may be created from dead edge @@ -265,7 +287,7 @@ func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node) } return sendLabelRefs(node.Namespace, bkt, fn) - } else if node.Type == ResourceSnapshot { + case ResourceSnapshot, resourceSnapshotFlat: parts := strings.SplitN(node.Key, "/", 2) if len(parts) != 2 { return errors.Errorf("invalid snapshot gc key %s", node.Key) @@ -280,11 +302,16 @@ func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node) } if pv := bkt.Get(bucketKeyParent); len(pv) > 0 { - fn(gcnode(ResourceSnapshot, node.Namespace, fmt.Sprintf("%s/%s", ss, pv))) + fn(gcnode(node.Type, node.Namespace, fmt.Sprintf("%s/%s", ss, pv))) + } + + // Do not send labeled references for flat snapshot refs + if node.Type == resourceSnapshotFlat { + return nil } return sendLabelRefs(node.Namespace, bkt, fn) - } else if node.Type == ResourceIngest { + case ResourceIngest: // Send expected value bkt := getBucket(tx, bucketKeyVersion, []byte(node.Namespace), bucketKeyObjectContent, bucketKeyObjectIngests, []byte(node.Key)) if bkt == nil { diff --git a/metadata/gc_test.go b/metadata/gc_test.go index eb1db7d05..f8cd99927 100644 --- a/metadata/gc_test.go +++ b/metadata/gc_test.go @@ -33,6 +33,15 @@ import ( bolt "go.etcd.io/bbolt" ) +func TestResourceMax(t *testing.T) { + if ResourceContent != resourceContentFlat&gc.ResourceMax { + t.Fatalf("Invalid flat content type: %d (max %d)", resourceContentFlat, gc.ResourceMax) + } + if ResourceSnapshot != resourceSnapshotFlat&gc.ResourceMax { + t.Fatalf("Invalid flat snapshot type: %d (max %d)", resourceSnapshotFlat, gc.ResourceMax) + } +} + func TestGCRoots(t *testing.T) { db, cleanup, err := newDatabase() if err != nil { @@ -90,6 +99,11 @@ func TestGCRoots(t *testing.T) { addLeaseSnapshot("ns2", "l4", "overlay", "sn8"), addLeaseIngest("ns2", "l4", "ingest-6"), addLeaseIngest("ns2", "l4", "ingest-7"), + + addLease("ns3", "l1", labelmap(string(labelGCFlat), time.Now().Add(time.Hour).Format(time.RFC3339))), + addLeaseContent("ns3", "l1", dgst(1)), + addLeaseSnapshot("ns3", "l1", "overlay", "sn1"), + addLeaseIngest("ns3", "l1", "ingest-1"), } expected := []gc.Node{ @@ -121,6 +135,10 @@ func TestGCRoots(t *testing.T) { gcnode(ResourceIngest, "ns1", "ingest-3"), gcnode(ResourceIngest, "ns2", "ingest-4"), gcnode(ResourceIngest, "ns2", "ingest-5"), + gcnode(ResourceLease, "ns3", "l1"), + gcnode(ResourceIngest, "ns3", "ingest-1"), + gcnode(resourceContentFlat, "ns3", dgst(1).String()), + gcnode(resourceSnapshotFlat, "ns3", "overlay/sn1"), } if err := db.Update(func(tx *bolt.Tx) error { @@ -268,6 +286,14 @@ func TestGCRefs(t *testing.T) { addSnapshot("ns2", "overlay", "sn3", "", labelmap( string(labelGCContentRef), dgst(1).String(), string(labelGCContentRef)+".keep-me", dgst(6).String())), + + // Test flat references don't follow label references + addContent("ns3", dgst(1), nil), + addContent("ns3", dgst(2), labelmap(string(labelGCContentRef)+".0", dgst(1).String())), + + addSnapshot("ns3", "overlay", "sn1", "", nil), + addSnapshot("ns3", "overlay", "sn2", "sn1", nil), + addSnapshot("ns3", "overlay", "sn3", "", labelmap(string(labelGCSnapRef)+"btrfs", "sn1", string(labelGCSnapRef)+"overlay", "sn1")), } refs := map[gc.Node][]gc.Node{ @@ -316,6 +342,18 @@ func TestGCRefs(t *testing.T) { gcnode(ResourceIngest, "ns2", "ingest-2"): { gcnode(ResourceContent, "ns2", dgst(8).String()), }, + gcnode(resourceSnapshotFlat, "ns3", "overlay/sn2"): { + gcnode(resourceSnapshotFlat, "ns3", "overlay/sn1"), + }, + gcnode(ResourceSnapshot, "ns3", "overlay/sn2"): { + gcnode(ResourceSnapshot, "ns3", "overlay/sn1"), + }, + gcnode(resourceSnapshotFlat, "ns3", "overlay/sn1"): nil, + gcnode(resourceSnapshotFlat, "ns3", "overlay/sn3"): nil, + gcnode(ResourceSnapshot, "ns3", "overlay/sn3"): { + gcnode(ResourceSnapshot, "ns3", "btrfs/sn1"), + gcnode(ResourceSnapshot, "ns3", "overlay/sn1"), + }, } if err := db.Update(func(tx *bolt.Tx) error {