github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

content: unify provider and ingester

Browse Source 
The split between provider and ingester was a long standing division
reflecting the client-side use cases. For the most part, we were
differentiating these for the algorithms that operate them, but it made
instantation and use of the types challenging. On the server-side, this
distinction is generally less important. This change unifies these types
and in the process we get a few benefits.

The first is that we now completely access the content store over GRPC.
This was the initial intent and we have now satisfied this goal
completely. There are a few issues around listing content and getting
status, but we resolve these with simple streaming and regexp filters.
More can probably be done to polish this but the result is clean.

Several other content-oriented methods were polished in the process of
unification. We have now properly seperated out the `Abort` method to
cancel ongoing or stalled ingest processes. We have also replaced the
`Active` method with a single status method.

The transition went extremely smoothly. Once the clients were updated to
use the new methods, every thing worked as expected on the first
compile.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day
2017-05-08 20:43:30 -07:00
parent 31cf34b726
commit 193abed96e
34 changed files with 1779 additions and 859 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
content/content.go
View File

@@ -28,24 +28,62 @@ var (
}
)
type Provider interface {
Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error)
}
type Ingester interface {
Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (Writer, error)
}
// TODO(stevvooe): Consider a very different name for this struct. Info is way
// to general. It also reads very weird in certain context, like pluralization.
type Info struct {
Digest digest.Digest
Size int64
CommittedAt time.Time
}
type Provider interface {
Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error)
}
type Status struct {
Ref string
Offset int64
Total int64
Expected digest.Digest
StartedAt time.Time
UpdatedAt time.Time
}
// WalkFunc defines the callback for a blob walk.
type WalkFunc func(Info) error
// Manager provides methods for inspecting, listing and removing content.
type Manager interface {
// Info will return metadata about content available in the content store.
//
// If the content is not present, ErrNotFound will be returned.
Info(ctx context.Context, dgst digest.Digest) (Info, error)
// Walk will call fn for each item in the content store.
Walk(ctx context.Context, fn WalkFunc) error
// Delete removes the content from the store.
Delete(ctx context.Context, dgst digest.Digest) error
// Status returns the status of any active ingestions whose ref match the
// provided regular expression. If empty, all active ingestions will be
// returned.
//
// TODO(stevvooe): Status may be slighly out of place here. If this remains
// here, we should remove Manager and just define these on store.
Status(ctx context.Context, re string) ([]Status, error)
// Abort completely cancels the ingest operation targeted by ref.
//
// TODO(stevvooe): Same consideration as above. This should really be
// restricted to an ingest management interface.
Abort(ctx context.Context, ref string) error
}
type Writer interface {
io.WriteCloser
Status() (Status, error)
@@ -54,8 +92,12 @@ type Writer interface {
Truncate(size int64) error
}
type Ingester interface {
Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (Writer, error)
// Store combines the methods of content-oriented interfaces into a set that
// are commonly provided by complete implementations.
type Store interface {
Manager
Ingester
Provider
}
func IsNotFound(err error) bool {

22
content/content_test.go
View File

@@ -52,7 +52,7 @@ func TestContentWriter(t *testing.T) {
}
// we should also see this as an active ingestion
ingestions, err := cs.Active()
ingestions, err := cs.Status(ctx, "")
if err != nil {
t.Fatal(err)
}
@@ -132,11 +132,9 @@ func TestWalkBlobs(t *testing.T) {
expected[dgst] = struct{}{}
}
if err := cs.Walk(func(path string, fi os.FileInfo, dgst digest.Digest) error {
found[dgst] = struct{}{}
if checked := checkBlobPath(t, cs, dgst); checked != path {
t.Fatalf("blob path did not match: %v != %v", path, checked)
}
if err := cs.Walk(ctx, func(bi Info) error {
found[bi.Digest] = struct{}{}
checkBlobPath(t, cs, bi.Digest)
return nil
}); err != nil {
t.Fatal(err)
@@ -203,7 +201,7 @@ func generateBlobs(t checker, nblobs, maxsize int64) map[digest.Digest][]byte {
return blobs
}
func populateBlobStore(t checker, ctx context.Context, cs *Store, nblobs, maxsize int64) map[digest.Digest][]byte {
func populateBlobStore(t checker, ctx context.Context, cs Store, nblobs, maxsize int64) map[digest.Digest][]byte {
blobs := generateBlobs(t, nblobs, maxsize)
for dgst, p := range blobs {
@@ -213,7 +211,7 @@ func populateBlobStore(t checker, ctx context.Context, cs *Store, nblobs, maxsiz
return blobs
}
func contentStoreEnv(t checker) (context.Context, string, *Store, func()) {
func contentStoreEnv(t checker) (context.Context, string, Store, func()) {
pc, _, _, ok := runtime.Caller(1)
if !ok {
t.Fatal("failed to resolve caller")
@@ -249,10 +247,10 @@ func checkCopy(t checker, size int64, dst io.Writer, src io.Reader) {
}
}
func checkBlobPath(t *testing.T, cs *Store, dgst digest.Digest) string {
path := cs.blobPath(dgst)
func checkBlobPath(t *testing.T, cs Store, dgst digest.Digest) string {
path := cs.(*store).blobPath(dgst)
if path != filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex()) {
if path != filepath.Join(cs.(*store).root, "blobs", dgst.Algorithm().String(), dgst.Hex()) {
t.Fatalf("unexpected path: %q", path)
}
fi, err := os.Stat(path)
@@ -268,7 +266,7 @@ func checkBlobPath(t *testing.T, cs *Store, dgst digest.Digest) string {
return path
}
func checkWrite(t checker, ctx context.Context, cs *Store, dgst digest.Digest, p []byte) digest.Digest {
func checkWrite(t checker, ctx context.Context, cs Store, dgst digest.Digest, p []byte) digest.Digest {
if err := WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(p), int64(len(p)), dgst); err != nil {
t.Fatal(err)
}

137
content/store.go
View File

@@ -7,6 +7,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strconv"
"time"
@@ -21,21 +22,21 @@ import (
//
// Store can generally support multi-reader, single-writer ingest of data,
// including resumable ingest.
type Store struct {
type store struct {
root string
}
func NewStore(root string) (*Store, error) {
func NewStore(root string) (Store, error) {
if err := os.MkdirAll(filepath.Join(root, "ingest"), 0777); err != nil && !os.IsExist(err) {
return nil, err
}
return &Store{
return &store{
root: root,
}, nil
}
func (s *Store) Info(dgst digest.Digest) (Info, error) {
func (s *store) Info(ctx context.Context, dgst digest.Digest) (Info, error) {
p := s.blobPath(dgst)
fi, err := os.Stat(p)
if err != nil {
@@ -46,11 +47,15 @@ func (s *Store) Info(dgst digest.Digest) (Info, error) {
return Info{}, err
}
return s.info(dgst, fi), nil
}
func (s *store) info(dgst digest.Digest, fi os.FileInfo) Info {
return Info{
Digest: dgst,
Size: fi.Size(),
CommittedAt: fi.ModTime(),
}, nil
}
}
// Open returns an io.ReadCloser for the blob.
@@ -58,7 +63,7 @@ func (s *Store) Info(dgst digest.Digest) (Info, error) {
// TODO(stevvooe): This would work much better as an io.ReaderAt in practice.
// Right now, we are doing type assertion to tease that out, but it won't scale
// well.
func (s *Store) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) {
func (s *store) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) {
fp, err := os.Open(s.blobPath(dgst))
if err != nil {
if os.IsNotExist(err) {
@@ -74,7 +79,7 @@ func (s *Store) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser,
//
// While this is safe to do concurrently, safe exist-removal logic must hold
// some global lock on the store.
func (cs *Store) Delete(dgst digest.Digest) error {
func (cs *store) Delete(ctx context.Context, dgst digest.Digest) error {
if err := os.RemoveAll(cs.blobPath(dgst)); err != nil {
if !os.IsNotExist(err) {
return err
@@ -88,14 +93,7 @@ func (cs *Store) Delete(dgst digest.Digest) error {
// TODO(stevvooe): Allow querying the set of blobs in the blob store.
// WalkFunc defines the callback for a blob walk.
//
// TODO(stevvooe): Remove the file info. Just need size and modtime. Perhaps,
// not a huge deal, considering we have a path, but let's not just let this one
// go without scrutiny.
type WalkFunc func(path string, fi os.FileInfo, dgst digest.Digest) error
func (cs *Store) Walk(fn WalkFunc) error {
func (cs *store) Walk(ctx context.Context, fn WalkFunc) error {
root := filepath.Join(cs.root, "blobs")
var alg digest.Algorithm
return filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
@@ -133,17 +131,60 @@ func (cs *Store) Walk(fn WalkFunc) error {
// store or extra paths not expected previously.
}
return fn(path, fi, dgst)
return fn(cs.info(dgst, fi))
})
}
// Status returns the current status of a blob by the ingest ref.
func (s *Store) Status(ref string) (Status, error) {
return s.status(s.ingestRoot(ref))
func (s *store) Status(ctx context.Context, re string) ([]Status, error) {
fp, err := os.Open(filepath.Join(s.root, "ingest"))
if err != nil {
return nil, err
}
defer fp.Close()
fis, err := fp.Readdir(-1)
if err != nil {
return nil, err
}
rec, err := regexp.Compile(re)
if err != nil {
return nil, err
}
var active []Status
for _, fi := range fis {
p := filepath.Join(s.root, "ingest", fi.Name())
stat, err := s.status(p)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// TODO(stevvooe): This is a common error if uploads are being
// completed while making this listing. Need to consider taking a
// lock on the whole store to coordinate this aspect.
//
// Another option is to cleanup downloads asynchronously and
// coordinate this method with the cleanup process.
//
// For now, we just skip them, as they really don't exist.
continue
}
if !rec.MatchString(stat.Ref) {
continue
}
active = append(active, stat)
}
return active, nil
}
// status works like stat above except uses the path to the ingest.
func (s *Store) status(ingestPath string) (Status, error) {
func (s *store) status(ingestPath string) (Status, error) {
dp := filepath.Join(ingestPath, "data")
fi, err := os.Stat(dp)
if err != nil {
@@ -165,7 +206,7 @@ func (s *Store) status(ingestPath string) (Status, error) {
}
// total attempts to resolve the total expected size for the write.
func (s *Store) total(ingestPath string) int64 {
func (s *store) total(ingestPath string) int64 {
totalS, err := readFileString(filepath.Join(ingestPath, "total"))
if err != nil {
return 0
@@ -185,7 +226,10 @@ func (s *Store) total(ingestPath string) int64 {
// ref at a time.
//
// The argument `ref` is used to uniquely identify a long-lived writer transaction.
func (s *Store) Writer(ctx context.Context, ref string, total int64, expected digest.Digest) (Writer, error) {
func (s *store) Writer(ctx context.Context, ref string, total int64, expected digest.Digest) (Writer, error) {
// TODO(stevvooe): Need to actually store and handle expected here. We have
// code in the service that shouldn't be dealing with this.
path, refp, data, lock, err := s.ingestPaths(ref)
if err != nil {
return nil, err
@@ -283,11 +327,11 @@ func (s *Store) Writer(ctx context.Context, ref string, total int64, expected di
// Abort an active transaction keyed by ref. If the ingest is active, it will
// be cancelled. Any resources associated with the ingest will be cleaned.
func (s *Store) Abort(ref string) error {
func (s *store) Abort(ctx context.Context, ref string) error {
root := s.ingestRoot(ref)
if err := os.RemoveAll(root); err != nil {
if os.IsNotExist(err) {
return nil
return ErrNotFound
}
return err
@@ -296,50 +340,11 @@ func (s *Store) Abort(ref string) error {
return nil
}
func (s *Store) Active() ([]Status, error) {
fp, err := os.Open(filepath.Join(s.root, "ingest"))
if err != nil {
return nil, err
}
defer fp.Close()
fis, err := fp.Readdir(-1)
if err != nil {
return nil, err
}
var active []Status
for _, fi := range fis {
p := filepath.Join(s.root, "ingest", fi.Name())
stat, err := s.status(p)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// TODO(stevvooe): This is a common error if uploads are being
// completed while making this listing. Need to consider taking a
// lock on the whole store to coordinate this aspect.
//
// Another option is to cleanup downloads asynchronously and
// coordinate this method with the cleanup process.
//
// For now, we just skip them, as they really don't exist.
continue
}
active = append(active, stat)
}
return active, nil
}
func (cs *Store) blobPath(dgst digest.Digest) string {
func (cs *store) blobPath(dgst digest.Digest) string {
return filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex())
}
func (s *Store) ingestRoot(ref string) string {
func (s *store) ingestRoot(ref string) string {
dgst := digest.FromString(ref)
return filepath.Join(s.root, "ingest", dgst.Hex())
}
@@ -351,7 +356,7 @@ func (s *Store) ingestRoot(ref string) string {
// - data: file where data is written
// - lock: lock file location
//
func (s *Store) ingestPaths(ref string) (string, string, string, lockfile.Lockfile, error) {
func (s *store) ingestPaths(ref string) (string, string, string, lockfile.Lockfile, error) {
var (
fp = s.ingestRoot(ref)
rp = filepath.Join(fp, "ref")

2
content/writer.go
View File

@@ -13,7 +13,7 @@ import (
// writer represents a write transaction against the blob store.
type writer struct {
s *Store
s *store
fp *os.File // opened data file
lock lockfile.Lockfile
path string // path to writer dir