diff --git a/.travis.yml b/.travis.yml index 221639cea..d7989fb2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,8 +49,6 @@ script: - export CGO_ENABLED=$TRAVIS_CGO_ENABLED - GIT_CHECK_EXCLUDE="./vendor" TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE/.../..}" make dco - make fmt -# FIXME: For non-linux GOOS, without running `go build -i`, vet fails with `vet: import failed: can't find import: fmt`... -# Note that `go build -i` requires write permission to GOROOT. (So it is not called in Makefile) - go build -i . - make setup - make lint diff --git a/Makefile b/Makefile index fac78f134..672c67d2b 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",) GO_LDFLAGS=-ldflags "-X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) $(EXTRA_LDFLAGS)" TESTFLAGS_RACE= +GO_GCFLAGS= #Detect the target os include Makefile.OS diff --git a/Makefile.darwin b/Makefile.darwin index 3011704bd..dd9c1ca3a 100644 --- a/Makefile.darwin +++ b/Makefile.darwin @@ -2,6 +2,6 @@ COMMANDS += containerd-shim # amd64 supports go test -race -ifeq ("amd64", $(GOARCH)) +ifeq ($(GOARCH),amd64) TESTFLAGS_RACE= -race endif diff --git a/Makefile.freebsd b/Makefile.freebsd index 4548eb1cd..a0a5ccbcf 100644 --- a/Makefile.freebsd +++ b/Makefile.freebsd @@ -2,6 +2,6 @@ COMMANDS += containerd-shim # amd64 supports go test -race -ifeq ("amd64", $(GOARCH)) +ifeq ($(GOARCH),amd64) TESTFLAGS_RACE= -race endif diff --git a/Makefile.linux b/Makefile.linux index 01b9ceb6c..7f297a912 100644 --- a/Makefile.linux +++ b/Makefile.linux @@ -1,7 +1,12 @@ #linux specific settings COMMANDS += containerd-shim +# check GOOS for cross compile builds +ifeq ($(GOOS),linux) + GO_GCFLAGS= -buildmode=pie +endif + # amd64 supports go test -race -ifeq ("amd64", $(GOARCH)) +ifeq ($(GOARCH),amd64) TESTFLAGS_RACE= -race endif diff --git a/Makefile.windows b/Makefile.windows index d41401211..f22ffeaaa 100644 --- a/Makefile.windows +++ b/Makefile.windows @@ -6,6 +6,6 @@ FIX_PATH = $(subst /,\,$1) BINARY_SUFFIX=".exe" # amd64 supports go test -race -ifeq ("amd64", $(GOARCH)) +ifeq ($(GOARCH),amd64) TESTFLAGS_RACE= -race endif diff --git a/gc/gc.go b/gc/gc.go index 47421a822..70838a762 100644 --- a/gc/gc.go +++ b/gc/gc.go @@ -145,10 +145,10 @@ func ConcurrentMark(ctx context.Context, root <-chan Node, refs func(context.Con // Sweep removes all nodes returned through the channel which are not in // the reachable set by calling the provided remove function. -func Sweep(reachable map[Node]struct{}, all <-chan Node, remove func(Node) error) error { +func Sweep(reachable map[Node]struct{}, all []Node, remove func(Node) error) error { // All black objects are now reachable, and all white objects are // unreachable. Free those that are white! - for node := range all { + for _, node := range all { if _, ok := reachable[node]; !ok { if err := remove(node); err != nil { return err diff --git a/metadata/db.go b/metadata/db.go index 18eba909e..8e9944876 100644 --- a/metadata/db.go +++ b/metadata/db.go @@ -199,42 +199,10 @@ func (m *DB) GarbageCollect(ctx context.Context) error { log.G(ctx).WithField("d", time.Now().Sub(lt1)).Debug("metadata garbage collected") }() - var marked map[gc.Node]struct{} - - if err := m.db.View(func(tx *bolt.Tx) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - roots := make(chan gc.Node) - errChan := make(chan error) - go func() { - defer close(errChan) - defer close(roots) - - // Call roots - if err := scanRoots(ctx, tx, roots); err != nil { - cancel() - errChan <- err - } - }() - - refs := func(ctx context.Context, n gc.Node, fn func(gc.Node)) error { - return references(ctx, tx, n, fn) - } - - reachable, err := gc.ConcurrentMark(ctx, roots, refs) - if rerr := <-errChan; rerr != nil { - return rerr - } - if err != nil { - return err - } - marked = reachable - return nil - }); err != nil { + marked, err := m.getMarked(ctx) + if err != nil { return err } - m.dirtyL.Lock() defer m.dirtyL.Unlock() @@ -242,15 +210,25 @@ func (m *DB) GarbageCollect(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - nodeC := make(chan gc.Node) - var scanErr error - + var ( + nodes []gc.Node + wg sync.WaitGroup + nodeC = make(chan gc.Node) + ) + wg.Add(1) go func() { - defer close(nodeC) - scanErr = scanAll(ctx, tx, nodeC) + defer wg.Done() + for n := range nodeC { + nodes = append(nodes, n) + } }() + if err := scanAll(ctx, tx, nodeC); err != nil { + return errors.Wrap(err, "failed to scan all") + } + close(nodeC) + wg.Wait() - rm := func(n gc.Node) error { + return gc.Sweep(marked, nodes, func(n gc.Node) error { if n.Type == ResourceSnapshot { if idx := strings.IndexRune(n.Key, '/'); idx > 0 { m.dirtySS[n.Key[:idx]] = struct{}{} @@ -259,17 +237,7 @@ func (m *DB) GarbageCollect(ctx context.Context) error { m.dirtyCS = true } return remove(ctx, tx, n) - } - - if err := gc.Sweep(marked, nodeC, rm); err != nil { - return errors.Wrap(err, "failed to sweep") - } - - if scanErr != nil { - return errors.Wrap(scanErr, "failed to scan all") - } - - return nil + }) }); err != nil { return err } @@ -293,6 +261,54 @@ func (m *DB) GarbageCollect(ctx context.Context) error { return nil } +func (m *DB) getMarked(ctx context.Context) (map[gc.Node]struct{}, error) { + var marked map[gc.Node]struct{} + if err := m.db.View(func(tx *bolt.Tx) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + nodes []gc.Node + wg sync.WaitGroup + roots = make(chan gc.Node) + ) + wg.Add(1) + go func() { + defer wg.Done() + for n := range roots { + nodes = append(nodes, n) + } + }() + // Call roots + if err := scanRoots(ctx, tx, roots); err != nil { + cancel() + return err + } + close(roots) + wg.Wait() + + refs := func(n gc.Node) ([]gc.Node, error) { + var sn []gc.Node + if err := references(ctx, tx, n, func(nn gc.Node) { + sn = append(sn, nn) + }); err != nil { + return nil, err + } + return sn, nil + } + + reachable, err := gc.Tricolor(nodes, refs) + if err != nil { + return err + } + marked = reachable + return nil + }); err != nil { + return nil, err + } + return marked, nil +} + func (m *DB) cleanupSnapshotter(name string) { ctx := context.Background() sn, ok := m.ss[name]