Metadata garbage collection

Marks and sweeps unreferenced objects.
Add snapshot cleanup to metadata.
Add content garbage collection

Add dirty flags for snapshotters and content store which
are set on deletion and used during the next garbage collection.
Cleanup content store backend when content metadata is removed.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

metadata/content.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/binary"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/boltdb/bolt"
@@ -11,6 +12,7 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/filters"
 	"github.com/containerd/containerd/labels"
+	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/metadata/boltutil"
 	"github.com/containerd/containerd/namespaces"
 	digest "github.com/opencontainers/go-digest"
@@ -19,12 +21,13 @@ import (
 
 type contentStore struct {
 	content.Store
-	db transactor
+	db *DB
+	l  sync.RWMutex
 }
 
 // newContentStore returns a namespaced content store using an existing
 // content store interface.
-func newContentStore(db transactor, cs content.Store) content.Store {
+func newContentStore(db *DB, cs content.Store) *contentStore {
 	return &contentStore{
 		Store: cs,
 		db:    db,
@@ -59,6 +62,9 @@ func (cs *contentStore) Update(ctx context.Context, info content.Info, fieldpath
 		return content.Info{}, err
 	}
 
+	cs.l.RLock()
+	defer cs.l.RUnlock()
+
 	updated := content.Info{
 		Digest: info.Digest,
 	}
@@ -166,15 +172,25 @@ func (cs *contentStore) Delete(ctx context.Context, dgst digest.Digest) error {
 		return err
 	}
 
+	cs.l.RLock()
+	defer cs.l.RUnlock()
+
 	return update(ctx, cs.db, func(tx *bolt.Tx) error {
 		bkt := getBlobBucket(tx, ns, dgst)
 		if bkt == nil {
 			return errors.Wrapf(errdefs.ErrNotFound, "content digest %v", dgst)
 		}
 
-		// Just remove local reference, garbage collector is responsible for
-		// cleaning up on disk content
-		return getBlobsBucket(tx, ns).DeleteBucket([]byte(dgst.String()))
+		if err := getBlobsBucket(tx, ns).DeleteBucket([]byte(dgst.String())); err != nil {
+			return err
+		}
+
+		// Mark content store as dirty for triggering garbage collection
+		cs.db.dirtyL.Lock()
+		cs.db.dirtyCS = true
+		cs.db.dirtyL.Unlock()
+
+		return nil
 	})
 }
 
@@ -269,6 +285,9 @@ func (cs *contentStore) Abort(ctx context.Context, ref string) error {
 		return err
 	}
 
+	cs.l.RLock()
+	defer cs.l.RUnlock()
+
 	return update(ctx, cs.db, func(tx *bolt.Tx) error {
 		bkt := getIngestBucket(tx, ns)
 		if bkt == nil {
@@ -293,6 +312,9 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
 		return nil, err
 	}
 
+	cs.l.RLock()
+	defer cs.l.RUnlock()
+
 	var w content.Writer
 	if err := update(ctx, cs.db, func(tx *bolt.Tx) error {
 		if expected != "" {
@@ -346,6 +368,7 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
 		ref:       ref,
 		namespace: ns,
 		db:        cs.db,
+		l:         &cs.l,
 	}, nil
 }
 
@@ -354,9 +377,13 @@ type namespacedWriter struct {
 	ref       string
 	namespace string
 	db        transactor
+	l         *sync.RWMutex
 }
 
 func (nw *namespacedWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
+	nw.l.RLock()
+	defer nw.l.RUnlock()
+
 	return update(ctx, nw.db, func(tx *bolt.Tx) error {
 		bkt := getIngestBucket(tx, nw.namespace)
 		if bkt != nil {
@@ -495,3 +522,61 @@ func writeInfo(info *content.Info, bkt *bolt.Bucket) error {
 
 	return bkt.Put(bucketKeySize, sizeEncoded)
 }
+
+func (cs *contentStore) garbageCollect(ctx context.Context) error {
+	lt1 := time.Now()
+	cs.l.Lock()
+	defer func() {
+		cs.l.Unlock()
+		log.G(ctx).WithField("t", time.Now().Sub(lt1)).Debugf("content garbage collected")
+	}()
+
+	seen := map[string]struct{}{}
+	if err := cs.db.View(func(tx *bolt.Tx) error {
+		v1bkt := tx.Bucket(bucketKeyVersion)
+		if v1bkt == nil {
+			return nil
+		}
+
+		// iterate through each namespace
+		v1c := v1bkt.Cursor()
+
+		for k, v := v1c.First(); k != nil; k, v = v1c.Next() {
+			if v != nil {
+				continue
+			}
+
+			cbkt := v1bkt.Bucket(k).Bucket(bucketKeyObjectContent)
+			if cbkt == nil {
+				continue
+			}
+			bbkt := cbkt.Bucket(bucketKeyObjectBlob)
+			if err := bbkt.ForEach(func(ck, cv []byte) error {
+				if cv == nil {
+					seen[string(ck)] = struct{}{}
+				}
+				return nil
+			}); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	if err := cs.Store.Walk(ctx, func(info content.Info) error {
+		if _, ok := seen[info.Digest.String()]; !ok {
+			if err := cs.Store.Delete(ctx, info.Digest); err != nil {
+				return err
+			}
+			log.G(ctx).WithField("digest", info.Digest).Debug("removed content")
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}