Replace lockfile with reference lock

Updates content service to handle lock errors and return
them to the client. The client remote handler has been
updated to retry when a resource is locked until the
resource is unlocked or the expected resource exists.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

remotes/handlers.go

@@ -3,6 +3,7 @@ package remotes
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/containerd/containerd/content"
@@ -55,17 +56,39 @@ func FetchHandler(ingester content.Ingester, fetcher Fetcher) images.HandlerFunc
 
 func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc ocispec.Descriptor) error {
 	log.G(ctx).Debug("fetch")
-	ref := MakeRefKey(ctx, desc)
 
-	cw, err := ingester.Writer(ctx, ref, desc.Size, desc.Digest)
-	if err != nil {
-		if !content.IsExists(err) {
-			return err
+	var (
+		ref   = MakeRefKey(ctx, desc)
+		cw    content.Writer
+		err   error
+		retry = 16
+	)
+	for {
+		cw, err = ingester.Writer(ctx, ref, desc.Size, desc.Digest)
+		if err != nil {
+			if content.IsExists(err) {
+				return nil
+			} else if !content.IsLocked(err) {
+				return err
+			}
+
+			// TODO: On first time locked is encountered, get status
+			// of writer and abort if not updated recently.
+
+			select {
+			case <-time.After(time.Millisecond * time.Duration(retry)):
+				if retry < 2048 {
+					retry = retry << 1
+				}
+				continue
+			case <-ctx.Done():
+				// Propagate lock error
+				return err
+			}
 		}
-
-		return nil
+		defer cw.Close()
+		break
 	}
-	defer cw.Close()
 
 	rc, err := fetcher.Fetch(ctx, desc)
 	if err != nil {