content/local: ensure that resumption is properly supported

While early PoCs had download resumption working, we didn't have tests
and had not verified the behavior. With this test suite, we now are able
to show that download resumption is properly supported in the content
store. In particular, there was a bug where resuming a download would
not issue the writes to the correct offset in the file. A Seek was added
to ensure we are writing from the current ingest offset.

In this investigation, it was also discovered that using the OS/Disk
created time on files was skewed from the monotonic clock in Go's
runtime. The startedat values are now taken from the Go runtime and
written to a separate file.

Signed-off-by: Stephen J Day <stephen.day@docker.com>

content/local/store.go

@@ -324,12 +324,25 @@ func (s *store) status(ingestPath string) (content.Status, error) {
 		return content.Status{}, err
 	}
 
+	startedAtP, err := ioutil.ReadFile(filepath.Join(ingestPath, "startedat"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = errors.Wrap(errdefs.ErrNotFound, err.Error())
+		}
+		return content.Status{}, err
+	}
+
+	var startedAt time.Time
+	if err := startedAt.UnmarshalText(startedAtP); err != nil {
+		return content.Status{}, errors.Wrapf(err, "could not parse startedat file")
+	}
+
 	return content.Status{
 		Ref:       ref,
 		Offset:    fi.Size(),
 		Total:     s.total(ingestPath),
 		UpdatedAt: fi.ModTime(),
-		StartedAt: getStartTime(fi),
+		StartedAt: startedAt,
 	}, nil
 }
 
@@ -412,7 +425,7 @@ func (s *store) Writer(ctx context.Context, ref string, total int64, expected di
 			return nil, errors.Errorf("provided total differs from status: %v != %v", total, status.Total)
 		}
 
-		// slow slow slow!!, send to goroutine or use resumable hashes
+		// TODO(stevvooe): slow slow slow!!, send to goroutine or use resumable hashes
 		fp, err := os.Open(data)
 		if err != nil {
 			return nil, err
@@ -431,20 +444,29 @@ func (s *store) Writer(ctx context.Context, ref string, total int64, expected di
 		startedAt = status.StartedAt
 		total = status.Total
 	} else {
+		startedAt = time.Now()
+		updatedAt = startedAt
+
 		// the ingest is new, we need to setup the target location.
 		// write the ref to a file for later use
 		if err := ioutil.WriteFile(refp, []byte(ref), 0666); err != nil {
 			return nil, err
 		}
 
+		startedAtP, err := startedAt.MarshalText()
+		if err != nil {
+			return nil, err
+		}
+
+		if err := ioutil.WriteFile(filepath.Join(path, "startedat"), startedAtP, 0666); err != nil {
+			return nil, err
+		}
+
 		if total > 0 {
 			if err := ioutil.WriteFile(filepath.Join(path, "total"), []byte(fmt.Sprint(total)), 0666); err != nil {
 				return nil, err
 			}
 		}
-
-		startedAt = time.Now()
-		updatedAt = startedAt
 	}
 
 	fp, err := os.OpenFile(data, os.O_WRONLY|os.O_CREATE, 0666)
@@ -452,6 +474,10 @@ func (s *store) Writer(ctx context.Context, ref string, total int64, expected di
 		return nil, errors.Wrap(err, "failed to open data file")
 	}
 
+	if _, err := fp.Seek(offset, io.SeekStart); err != nil {
+		return nil, errors.Wrap(err, "could not seek to current write offset")
+	}
+
 	return &writer{
 		s:         s,
 		fp:        fp,