pushWriter: correctly propagate errors

In the refactor from 926b9c72f61b5be6bf8d952512f1d0932fbaf898, the error
handling was substantially reworked, and changed the types of errors
returned.

Notably, in the case of a network error, instead of propogating the
error through to return from pushWriter.Write (as previously), it would
be propagated through to pushWriter.Commit - however, this is too late,
since we've already closed the io.Pipe by the time we would have reached
this function. Therefore, we get the generic error message  "io:
read/write on closed pipe" for *every network error*.

This patch corrects this behavior to ensure that the correct error
object is always returned as early as possible, by checking the error
result after writing and detecting a closed pipe.

Additionally, we do some additional hardening - specifically we prevent
falling through when resetting the content or detecting errors, and
update the tests to explicitly check for the ErrReset message.

Signed-off-by: Justin Chadwell <me@jedevc.com>

remotes/docker/pusher_test.go

@@ -117,26 +117,19 @@ func TestPusherErrReset(t *testing.T) {
 	}
 
 	w, err := p.push(context.Background(), desc, remotes.MakeRefKey(context.Background(), desc), false)
-	assert.Equal(t, err, nil, "no error should be there")
+	assert.NoError(t, err)
 
-	w.Write(ct)
+	// first push should fail with ErrReset
+	_, err = w.Write(ct)
+	assert.NoError(t, err)
+	err = w.Commit(context.Background(), desc.Size, desc.Digest)
+	assert.Equal(t, content.ErrReset, err)
 
-	pw, _ := w.(*pushWriter)
-
-	select {
-	case p := <-pw.pipeC:
-		p.Write(ct)
-	case e := <-pw.errC:
-		assert.Failf(t, "error: %v while retrying request", e.Error())
-	}
-
-	select {
-	case resp := <-pw.respC:
-		assert.Equalf(t, resp.StatusCode, http.StatusCreated,
-			"201 should be the response code when uploading new content")
-	case <-pw.errC:
-		assert.Fail(t, "should not give error")
-	}
+	// second push should succeed
+	_, err = w.Write(ct)
+	assert.NoError(t, err)
+	err = w.Commit(context.Background(), desc.Size, desc.Digest)
+	assert.NoError(t, err)
 }
 
 func tryUpload(ctx context.Context, t *testing.T, p dockerPusher, layerContent []byte) error {