From b674c64827f99d1a1c1e131550e5323013d528cf Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 21 Nov 2017 15:45:09 -0800 Subject: [PATCH] Support unknown size from schema 1 manifests Schema1 manifests did not set a size in the digest for the blobs, breaking the expectations of the update http seeking reader. Now the http seeker has been updated to support unknown size as a value of negative 1 and the schema1 puller sets the unknown size accordingly. Signed-off-by: Derek McGowan --- remotes/docker/httpreadseeker.go | 5 ++++- remotes/docker/schema1/converter.go | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/remotes/docker/httpreadseeker.go b/remotes/docker/httpreadseeker.go index b042a8852..f6de60a27 100644 --- a/remotes/docker/httpreadseeker.go +++ b/remotes/docker/httpreadseeker.go @@ -64,6 +64,9 @@ func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) { case io.SeekCurrent: abs += offset case io.SeekEnd: + if hrs.size == -1 { + return 0, errors.Wrap(errdefs.ErrUnavailable, "Fetcher.Seek: unknown size, cannot seek from end") + } abs = hrs.size + offset default: return 0, errors.Wrap(errdefs.ErrInvalidArgument, "Fetcher.Seek: invalid whence") @@ -93,7 +96,7 @@ func (hrs *httpReadSeeker) reader() (io.Reader, error) { return hrs.rc, nil } - if hrs.offset < hrs.size { + if hrs.size == -1 || hrs.offset < hrs.size { // only try to reopen the body request if we are seeking to a value // less than the actual size. if hrs.open == nil { diff --git a/remotes/docker/schema1/converter.go b/remotes/docker/schema1/converter.go index dfc58e322..6b74cd67e 100644 --- a/remotes/docker/schema1/converter.go +++ b/remotes/docker/schema1/converter.go @@ -83,6 +83,7 @@ func (c *Converter) Handle(ctx context.Context, desc ocispec.Descriptor) ([]ocis { MediaType: images.MediaTypeDockerSchema2LayerGzip, Digest: c.pulledManifest.FSLayers[i].BlobSum, + Size: -1, }, }, descs...) } @@ -209,10 +210,16 @@ func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) erro ref = remotes.MakeRefKey(ctx, desc) calc = newBlobStateCalculator() retry = 16 + size = desc.Size ) + // size may be unknown, set to zero for content ingest + if size == -1 { + size = 0 + } + tryit: - cw, err := c.contentStore.Writer(ctx, ref, desc.Size, desc.Digest) + cw, err := c.contentStore.Writer(ctx, ref, size, desc.Digest) if err != nil { if errdefs.IsUnavailable(err) { select { @@ -273,7 +280,8 @@ tryit: eg.Go(func() error { defer pw.Close() - return content.Copy(ctx, cw, io.TeeReader(rc, pw), desc.Size, desc.Digest) + + return content.Copy(ctx, cw, io.TeeReader(rc, pw), size, desc.Digest) }) if err := eg.Wait(); err != nil { @@ -281,7 +289,7 @@ tryit: } } - if desc.Size == 0 { + if desc.Size == -1 { info, err := c.contentStore.Info(ctx, desc.Digest) if err != nil { return errors.Wrap(err, "failed to get blob info")