Add writer open helper to handle unavailable refs

Updates blob writer helper to use new open and ensure
unavailable errors are always handled.
Removes duplication of unavailable handling code.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

content/helpers.go

@@ -20,7 +20,9 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"sync"
+	"time"
 
 	"github.com/containerd/containerd/errdefs"
 	"github.com/opencontainers/go-digest"
@@ -64,7 +66,7 @@ func ReadBlob(ctx context.Context, provider Provider, dgst digest.Digest) ([]byt
 //
 // Copy is buffered, so no need to wrap reader in buffered io.
 func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error {
-	cw, err := cs.Writer(ctx, ref, size, expected)
+	cw, err := OpenWriter(ctx, cs, ref, size, expected)
 	if err != nil {
 		if !errdefs.IsAlreadyExists(err) {
 			return err
@@ -77,6 +79,43 @@ func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, size i
 	return Copy(ctx, cw, r, size, expected, opts...)
 }
 
+// OpenWriter opens a new writer for the given reference, retrying if the writer
+// is locked until the reference is available or returns an error.
+func OpenWriter(ctx context.Context, cs Ingester, ref string, size int64, expected digest.Digest) (Writer, error) {
+	var (
+		cw    Writer
+		err   error
+		retry = 16
+	)
+	for {
+		cw, err = cs.Writer(ctx, ref, size, expected)
+		if err != nil {
+			if !errdefs.IsUnavailable(err) {
+				return nil, err
+			}
+
+			// TODO: Check status to determine if the writer is active,
+			// continue waiting while active, otherwise return lock
+			// error or abort. Requires asserting for an ingest manager
+
+			select {
+			case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))):
+				if retry < 2048 {
+					retry = retry << 1
+				}
+				continue
+			case <-ctx.Done():
+				// Propagate lock error
+				return nil, err
+			}
+
+		}
+		break
+	}
+
+	return cw, err
+}
+
 // Copy copies data with the expected digest from the reader into the
 // provided content store writer. This copy commits the writer.
 //