github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Fix writer deadlock in local store

Browse Source 
The local store could end up in a state where the writer is
closed but the reference is locked after a commit on an
existing object.
Cleans up Commit logic to always close the writer even after
an error occurs, guaranteeing the reference is unlocked after commit.
Adds a test to the content test suite to verify this behavior.
Updates the content store interface definitions to clarify the behavior.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
This commit is contained in:
Derek McGowan
2018-10-10 17:00:07 -07:00
parent 15f19d7a67
commit 0f756495a9
6 changed files with 240 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
content/local/store.go
View File

@@ -524,12 +524,11 @@ func (s *store) writer(ctx context.Context, ref string, total int64, expected di
if err != nil {
return nil, err
}
defer fp.Close()
p := bufPool.Get().(*[]byte)
defer bufPool.Put(p)
offset, err = io.CopyBuffer(digester.Hash(), fp, *p)
bufPool.Put(p)
fp.Close()
if err != nil {
return nil, err
}

71
content/local/writer.go
View File

@@ -26,6 +26,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/log"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
@@ -80,43 +81,36 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest,
}
}
if w.fp == nil {
// Ensure even on error the writer is fully closed
defer unlock(w.ref)
fp := w.fp
w.fp = nil
if fp == nil {
return errors.Wrap(errdefs.ErrFailedPrecondition, "cannot commit on closed writer")
}
if err := w.fp.Sync(); err != nil {
if err := fp.Sync(); err != nil {
fp.Close()
return errors.Wrap(err, "sync failed")
}
fi, err := w.fp.Stat()
fi, err := fp.Stat()
closeErr := fp.Close()
if err != nil {
return errors.Wrap(err, "stat on ingest file failed")
}
// change to readonly, more important for read, but provides _some_
// protection from this point on. We use the existing perms with a mask
// only allowing reads honoring the umask on creation.
//
// This removes write and exec, only allowing read per the creation umask.
//
// NOTE: Windows does not support this operation
if runtime.GOOS != "windows" {
if err := w.fp.Chmod((fi.Mode() & os.ModePerm) &^ 0333); err != nil {
return errors.Wrap(err, "failed to change ingest file permissions")
}
if closeErr != nil {
return errors.Wrap(err, "failed to close ingest file")
}
if size > 0 && size != fi.Size() {
return errors.Errorf("unexpected commit size %d, expected %d", fi.Size(), size)
}
if err := w.fp.Close(); err != nil {
return errors.Wrap(err, "failed closing ingest")
return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit size %d, expected %d", fi.Size(), size)
}
dgst := w.digester.Digest()
if expected != "" && expected != dgst {
return errors.Errorf("unexpected commit digest %s, expected %s", dgst, expected)
return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit digest %s, expected %s", dgst, expected)
}
var (
@@ -129,27 +123,48 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest,
return err
}
// clean up!!
defer os.RemoveAll(w.path)
if _, err := os.Stat(target); err == nil {
// collision with the target file!
if err := os.RemoveAll(w.path); err != nil {
log.G(ctx).WithField("ref", w.ref).WithField("path", w.path).Errorf("failed to remove ingest directory")
}
return errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", dgst)
}
if err := os.Rename(ingest, target); err != nil {
return err
}
// Ingest has now been made available in the content store, attempt to complete
// setting metadata but errors should only be logged and not returned since
// the content store cannot be cleanly rolled back.
commitTime := time.Now()
if err := os.Chtimes(target, commitTime, commitTime); err != nil {
return err
log.G(ctx).WithField("digest", dgst).Errorf("failed to change file time to commit time")
}
w.fp = nil
unlock(w.ref)
// clean up!!
if err := os.RemoveAll(w.path); err != nil {
log.G(ctx).WithField("ref", w.ref).WithField("path", w.path).Errorf("failed to remove ingest directory")
}
if w.s.ls != nil && base.Labels != nil {
if err := w.s.ls.Set(dgst, base.Labels); err != nil {
return err
log.G(ctx).WithField("digest", dgst).Errorf("failed to set labels")
}
}
// change to readonly, more important for read, but provides _some_
// protection from this point on. We use the existing perms with a mask
// only allowing reads honoring the umask on creation.
//
// This removes write and exec, only allowing read per the creation umask.
//
// NOTE: Windows does not support this operation
if runtime.GOOS != "windows" {
if err := os.Chmod(target, (fi.Mode()&os.ModePerm)&^0333); err != nil {
log.G(ctx).WithField("ref", w.ref).Errorf("failed to make readonly")
}
}