From 8eee47cf825c70cca83ee774a6cb2ce332e6f638 Mon Sep 17 00:00:00 2001
From: Stephen J Day <stephen.day@docker.com>
Date: Mon, 30 Oct 2017 15:40:54 -0700
Subject: [PATCH] remotes/docker/schema1: back off on locked ref

To allow concurrent pull of images of the v1 persuasion, we need to
backoff when multiple pullers are trying to operate on the same
resource. The back off logic is ported to v1 pull to match the behavior
for other images.

A little randomness is also added to the backoff to prevent thundering
herd and to reduce expected recovery time.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
---
 remotes/docker/schema1/converter.go | 22 ++++++++++++++++++----
 remotes/handlers.go                 |  3 ++-
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/remotes/docker/schema1/converter.go b/remotes/docker/schema1/converter.go
index 99940c88a..6e0dcb4e2 100644
--- a/remotes/docker/schema1/converter.go
+++ b/remotes/docker/schema1/converter.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"strings"
 	"sync"
 	"time"
@@ -215,13 +216,26 @@ func (c *Converter) fetchManifest(ctx context.Context, desc ocispec.Descriptor)
 func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) error {
 	log.G(ctx).Debug("fetch blob")
 
-	ref := remotes.MakeRefKey(ctx, desc)
-
-	calc := newBlobStateCalculator()
+	var (
+		ref   = remotes.MakeRefKey(ctx, desc)
+		calc  = newBlobStateCalculator()
+		retry = 16
+	)
 
+tryit:
 	cw, err := c.contentStore.Writer(ctx, ref, desc.Size, desc.Digest)
 	if err != nil {
-		if !errdefs.IsAlreadyExists(err) {
+		if errdefs.IsUnavailable(err) {
+			select {
+			case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))):
+				if retry < 2048 {
+					retry = retry << 1
+				}
+				goto tryit
+			case <-ctx.Done():
+				return err
+			}
+		} else if !errdefs.IsAlreadyExists(err) {
 			return err
 		}
 
diff --git a/remotes/handlers.go b/remotes/handlers.go
index e6d213299..7b1dad99e 100644
--- a/remotes/handlers.go
+++ b/remotes/handlers.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math/rand"
 	"time"
 
 	"github.com/containerd/containerd/content"
@@ -84,7 +85,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc
 			// of writer and abort if not updated recently.
 
 			select {
-			case <-time.After(time.Millisecond * time.Duration(retry)):
+			case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))):
 				if retry < 2048 {
 					retry = retry << 1
 				}