From 963a01735bb1c94d95807f5e2ffc1f9e7a5854e2 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Thu, 23 Aug 2018 21:34:59 -0700 Subject: [PATCH 1/2] Add timeout for container/sandbox recover and event monitor. Signed-off-by: Lantao Liu --- pkg/server/events.go | 13 +++++++++++-- pkg/server/restart.go | 23 ++++++++++++++++++++--- pkg/store/image/fake_image.go | 2 +- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/pkg/server/events.go b/pkg/server/events.go index c3ab1dfd5..e6e0321b6 100644 --- a/pkg/server/events.go +++ b/pkg/server/events.go @@ -42,11 +42,17 @@ const ( backOffInitDuration = 1 * time.Second backOffMaxDuration = 5 * time.Minute backOffExpireCheckDuration = 1 * time.Second + + // handleEventTimeout is the timeout for handling 1 event. Event monitor + // handles events in serial, if one event blocks the event monitor, no + // other events can be handled. + // Add a timeout for each event handling, events that timeout will be requeued and + // handled again in the future. + handleEventTimeout = 10 * time.Second ) // eventMonitor monitors containerd event and updates internal state correspondingly. -// TODO(random-liu): [P1] Figure out is it possible to drop event during containerd -// is running. If it is, we should do periodically list to sync state with containerd. +// TODO(random-liu): Handle event for each container in a separate goroutine. type eventMonitor struct { containerStore *containerstore.Store sandboxStore *sandboxstore.Store @@ -189,6 +195,9 @@ func (em *eventMonitor) stop() { // handleEvent handles a containerd event. func (em *eventMonitor) handleEvent(any interface{}) error { ctx := ctrdutil.NamespacedContext() + ctx, cancel := context.WithTimeout(ctx, handleEventTimeout) + defer cancel() + switch any.(type) { // If containerd-shim exits unexpectedly, there will be no corresponding event. // However, containerd could not retrieve container state in that case, so it's diff --git a/pkg/server/restart.go b/pkg/server/restart.go index 560c75f0f..bd25d152a 100644 --- a/pkg/server/restart.go +++ b/pkg/server/restart.go @@ -136,8 +136,23 @@ func (c *criService) recover(ctx context.Context) error { return nil } +// loadContainerTimeout is the default timeout for loading a container/sandbox. +// One container/sandbox hangs (e.g. containerd#2438) should not affect other +// containers/sandboxes. +// Most CRI container/sandbox related operations are per container, the ones +// which handle multiple containers at a time are: +// * ListPodSandboxes: Don't talk with containerd services. +// * ListContainers: Don't talk with containerd services. +// * ListContainerStats: Not in critical code path, a default timeout will +// be applied at CRI level. +// * Recovery logic: We should set a time for each container/sandbox recovery. +// * Event montior: We should set a timeout for each container/sandbox event handling. +const loadContainerTimeout = 10 * time.Second + // loadContainer loads container from containerd and status checkpoint. func (c *criService) loadContainer(ctx context.Context, cntr containerd.Container) (containerstore.Container, error) { + ctx, cancel := context.WithTimeout(ctx, loadContainerTimeout) + defer cancel() id := cntr.ID() containerDir := c.getContainerRootDir(id) volatileContainerDir := c.getVolatileContainerRootDir(id) @@ -290,9 +305,9 @@ const ( // unknownContainerStatus returns the default container status when its status is unknown. func unknownContainerStatus() containerstore.Status { return containerstore.Status{ - CreatedAt: time.Now().UnixNano(), - StartedAt: time.Now().UnixNano(), - FinishedAt: time.Now().UnixNano(), + CreatedAt: 0, + StartedAt: 0, + FinishedAt: 0, ExitCode: unknownExitCode, Reason: unknownExitReason, } @@ -300,6 +315,8 @@ func unknownContainerStatus() containerstore.Status { // loadSandbox loads sandbox from containerd. func loadSandbox(ctx context.Context, cntr containerd.Container) (sandboxstore.Sandbox, error) { + ctx, cancel := context.WithTimeout(ctx, loadContainerTimeout) + defer cancel() var sandbox sandboxstore.Sandbox // Load sandbox metadata. exts, err := cntr.Extensions(ctx) diff --git a/pkg/store/image/fake_image.go b/pkg/store/image/fake_image.go index b82d5c408..6d74d2417 100644 --- a/pkg/store/image/fake_image.go +++ b/pkg/store/image/fake_image.go @@ -27,7 +27,7 @@ func NewFakeStore(images []Image) (*Store, error) { s.refCache[ref] = i.ID } if err := s.store.add(i); err != nil { - return nil, errors.Wrapf(err, "add image %q", i) + return nil, errors.Wrapf(err, "add image %+v", i) } } return s, nil From e3aa8adbb0cd8ad9cbdba4622d4b34b3a4a0e9f4 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Fri, 21 Sep 2018 14:36:56 -0700 Subject: [PATCH 2/2] Update containerd to f88d3e5d6dfe9b7d7941ac5241649ad8240b9282. Signed-off-by: Lantao Liu --- vendor.conf | 12 +- .../Microsoft/hcsshim/internal/guid/guid.go | 47 ++++ .../Microsoft/hcsshim/internal/hcs/process.go | 7 + .../Microsoft/hcsshim/internal/hcs/system.go | 48 +++- .../Microsoft/hcsshim/internal/hcs/watcher.go | 30 +++ .../internal/interop/zsyscall_windows.go | 2 +- .../internal/safefile/zsyscall_windows.go | 2 +- .../hcsshim/internal/timeout/timeout.go | 62 ++++- .../hcsshim/internal/wclayer/createlayer.go | 6 +- .../containerd/containerd/README.md | 2 +- .../api/services/content/v1/content.pb.go | 4 +- .../api/services/content/v1/content.proto | 2 +- .../archive/compression/compression.go | 88 +++++- .../cmd/containerd/command/publish.go | 2 +- .../containerd/cmd/ctr/commands/commands.go | 4 + .../cmd/ctr/commands/images/import.go | 72 +++-- .../containerd/cmd/ctr/commands/run/run.go | 4 + .../cmd/ctr/commands/run/run_unix.go | 8 +- .../cmd/ctr/commands/run/run_windows.go | 95 ++++--- .../containerd/containers/containers.go | 4 +- .../containerd/containerd/content/helpers.go | 6 +- .../containerd/content/local/writer.go | 8 +- .../content/proxy/content_writer.go | 6 +- .../containerd/diff/walking/differ.go | 4 +- .../containerd/containerd/export.go | 3 +- .../github.com/containerd/containerd/image.go | 6 + .../containerd/images/archive/importer.go | 254 ++++++++++++++++++ .../containerd/images/archive/reference.go | 86 ++++++ .../containerd/images/importexport.go | 2 +- .../containerd/images/oci/exporter.go | 4 +- .../containerd/images/oci/importer.go | 204 -------------- .../containerd/containerd/import.go | 133 +++++++-- .../containerd/containerd/metadata/bolt.go | 2 +- .../containerd/metadata/boltutil/helpers.go | 2 +- .../containerd/containerd/metadata/buckets.go | 6 +- .../containerd/metadata/containers.go | 2 +- .../containerd/containerd/metadata/content.go | 17 +- .../containerd/containerd/metadata/db.go | 4 +- .../containerd/containerd/metadata/gc.go | 2 +- .../containerd/containerd/metadata/images.go | 2 +- .../containerd/containerd/metadata/leases.go | 2 +- .../containerd/metadata/migrations.go | 14 +- .../containerd/metadata/namespaces.go | 2 +- .../containerd/metadata/snapshot.go | 2 +- .../containerd/containerd/oci/spec.go | 1 + .../containerd/containerd/oci/spec_opts.go | 27 +- .../containerd/runtime/v1/linux/bundle.go | 7 +- .../containerd/runtime/v1/linux/runtime.go | 2 +- .../containerd/runtime/v2/manager.go | 2 +- .../containerd/services/containers/local.go | 2 +- .../containerd/services/content/service.go | 6 +- .../containerd/services/leases/local.go | 2 +- .../containerd/services/namespaces/local.go | 2 +- .../containerd/services/server/server.go | 2 +- .../containerd/services/tasks/local.go | 2 +- .../containerd/snapshots/storage/bolt.go | 2 +- .../containerd/snapshots/storage/metastore.go | 2 +- .../github.com/containerd/containerd/task.go | 5 +- .../containerd/containerd/vendor.conf | 41 +-- .../containerd/containerd/version/version.go | 2 +- .../containerd/continuity/context.go | 16 ++ .../containerd/continuity/devices/devices.go | 16 ++ .../continuity/devices/devices_unix.go | 16 ++ .../continuity/devices/devices_windows.go | 16 ++ .../containerd/continuity/digests.go | 16 ++ .../containerd/continuity/driver/driver.go | 16 ++ .../continuity/driver/driver_unix.go | 28 +- .../continuity/driver/driver_windows.go | 21 +- .../continuity/driver/lchmod_linux.go | 22 +- .../continuity/driver/lchmod_unix.go | 22 +- .../containerd/continuity/driver/utils.go | 16 ++ .../containerd/continuity/fs/copy.go | 16 ++ .../containerd/continuity/fs/copy_linux.go | 16 ++ .../containerd/continuity/fs/copy_unix.go | 16 ++ .../containerd/continuity/fs/copy_windows.go | 16 ++ .../containerd/continuity/fs/diff.go | 16 ++ .../containerd/continuity/fs/diff_unix.go | 16 ++ .../containerd/continuity/fs/diff_windows.go | 16 ++ .../containerd/continuity/fs/dtype_linux.go | 16 ++ .../github.com/containerd/continuity/fs/du.go | 16 ++ .../containerd/continuity/fs/du_unix.go | 16 ++ .../containerd/continuity/fs/du_windows.go | 16 ++ .../containerd/continuity/fs/hardlink.go | 16 ++ .../containerd/continuity/fs/hardlink_unix.go | 16 ++ .../continuity/fs/hardlink_windows.go | 16 ++ .../containerd/continuity/fs/path.go | 22 +- .../containerd/continuity/fs/stat_bsd.go | 16 ++ .../containerd/continuity/fs/stat_linux.go | 16 ++ .../containerd/continuity/fs/time.go | 16 ++ .../containerd/continuity/groups_unix.go | 16 ++ .../containerd/continuity/hardlinks.go | 16 ++ .../containerd/continuity/hardlinks_unix.go | 16 ++ .../continuity/hardlinks_windows.go | 16 ++ .../containerd/continuity/ioutils.go | 16 ++ .../containerd/continuity/manifest.go | 16 ++ .../continuity/pathdriver/path_driver.go | 16 ++ .../containerd/continuity/proto/gen.go | 16 ++ .../containerd/continuity/resource.go | 16 ++ .../containerd/continuity/resource_unix.go | 16 ++ .../containerd/continuity/resource_windows.go | 16 ++ .../continuity/syscallx/syscall_unix.go | 16 ++ .../continuity/syscallx/syscall_windows.go | 16 ++ .../containerd/continuity/sysx/file_posix.go | 16 ++ .../continuity/sysx/nodata_linux.go | 16 ++ .../continuity/sysx/nodata_solaris.go | 16 ++ .../containerd/continuity/sysx/nodata_unix.go | 16 ++ .../containerd/continuity/sysx/xattr.go | 16 ++ .../continuity/sysx/xattr_unsupported.go | 16 ++ vendor/github.com/containerd/ttrpc/client.go | 4 + vendor/github.com/containerd/ttrpc/server.go | 2 +- .../github.com/opencontainers/runc/README.md | 12 + .../runc/libcontainer/nsenter/README.md | 6 +- .../runc/libcontainer/nsenter/nsexec.c | 2 +- .../boltdb/bolt => go.etcd.io/bbolt}/LICENSE | 0 .../bolt => go.etcd.io/bbolt}/README.md | 189 +++++++------ .../bolt => go.etcd.io/bbolt}/bolt_386.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_amd64.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_arm.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_arm64.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_linux.go | 2 +- vendor/go.etcd.io/bbolt/bolt_mips64x.go | 12 + vendor/go.etcd.io/bbolt/bolt_mipsx.go | 12 + .../bolt => go.etcd.io/bbolt}/bolt_openbsd.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_ppc.go | 5 +- .../bolt => go.etcd.io/bbolt}/bolt_ppc64.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_ppc64le.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_s390x.go | 2 +- .../bolt => go.etcd.io/bbolt}/bolt_unix.go | 42 +-- .../bbolt}/bolt_unix_solaris.go | 44 ++- .../bolt => go.etcd.io/bbolt}/bolt_windows.go | 57 ++-- .../bbolt}/boltsync_unix.go | 2 +- .../bolt => go.etcd.io/bbolt}/bucket.go | 16 +- .../bolt => go.etcd.io/bbolt}/cursor.go | 10 +- .../boltdb/bolt => go.etcd.io/bbolt}/db.go | 221 ++++++++++----- .../boltdb/bolt => go.etcd.io/bbolt}/doc.go | 4 +- .../bolt => go.etcd.io/bbolt}/errors.go | 2 +- .../bolt => go.etcd.io/bbolt}/freelist.go | 133 +++++++-- .../boltdb/bolt => go.etcd.io/bbolt}/node.go | 4 +- .../boltdb/bolt => go.etcd.io/bbolt}/page.go | 2 +- .../boltdb/bolt => go.etcd.io/bbolt}/tx.go | 85 +++--- 140 files changed, 2342 insertions(+), 726 deletions(-) create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go create mode 100644 vendor/github.com/containerd/containerd/images/archive/importer.go create mode 100644 vendor/github.com/containerd/containerd/images/archive/reference.go delete mode 100644 vendor/github.com/containerd/containerd/images/oci/importer.go rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/LICENSE (100%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/README.md (90%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_386.go (95%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_amd64.go (95%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_arm.go (98%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_arm64.go (95%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_linux.go (91%) create mode 100644 vendor/go.etcd.io/bbolt/bolt_mips64x.go create mode 100644 vendor/go.etcd.io/bbolt/bolt_mipsx.go rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_openbsd.go (97%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_ppc.go (69%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_ppc64.go (95%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_ppc64le.go (95%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_s390x.go (95%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_unix.go (71%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_unix_solaris.go (70%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bolt_windows.go (76%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/boltsync_unix.go (91%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/bucket.go (99%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/cursor.go (99%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/db.go (84%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/doc.go (94%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/errors.go (99%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/freelist.go (65%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/node.go (99%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/page.go (99%) rename vendor/{github.com/boltdb/bolt => go.etcd.io/bbolt}/tx.go (94%) diff --git a/vendor.conf b/vendor.conf index 269b8c3c3..37ab367c2 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,15 +1,14 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 github.com/blang/semver v3.1.0 -github.com/boltdb/bolt v1.3.1 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2 github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23 -github.com/containerd/containerd 1950f791d9225ffe061c77e74e292bcb3c428a04 -github.com/containerd/continuity f44b615e492bdfb371aae2f76ec694d9da1db537 +github.com/containerd/containerd f88d3e5d6dfe9b7d7941ac5241649ad8240b9282 +github.com/containerd/continuity 7f53d412b9eb1cbf744c2063185d703a0ee34700 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/go-cni 6d7b509a054a3cb1c35ed1865d4fde2f0cb547cd github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3 -github.com/containerd/ttrpc 94dde388801693c54f88a6596f713b51a8b30b2d +github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 github.com/containernetworking/cni v0.6.0 github.com/containernetworking/plugins v0.7.0 @@ -35,12 +34,12 @@ github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f github.com/json-iterator/go 1.1.5 github.com/matttproud/golang_protobuf_extensions v1.0.0 github.com/Microsoft/go-winio v0.4.10 -github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55 +github.com/Microsoft/hcsshim v0.7.4 github.com/modern-go/concurrent 1.0.3 github.com/modern-go/reflect2 1.0.1 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 github.com/opencontainers/image-spec v1.0.1 -github.com/opencontainers/runc 20aff4f0488c6d4b8df4d85b4f63f1f704c11abd +github.com/opencontainers/runc 00dc70017d222b178a002ed30e9321b12647af2d github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 github.com/opencontainers/runtime-tools fb101d5d42ab9c040f7d0a004e78336e5d5cb197 github.com/opencontainers/selinux b6fa367ed7f534f9ba25391cc2d467085dbb445a @@ -59,6 +58,7 @@ github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874 +go.etcd.io/bbolt v1.3.1-etcd.8 golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4 diff --git a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go b/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go index c37dec8c7..e9e45c030 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go @@ -2,10 +2,16 @@ package guid import ( "crypto/rand" + "encoding/json" "fmt" "io" + "strconv" + "strings" ) +var _ = (json.Marshaler)(&GUID{}) +var _ = (json.Unmarshaler)(&GUID{}) + type GUID [16]byte func New() GUID { @@ -20,3 +26,44 @@ func New() GUID { func (g GUID) String() string { return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:]) } + +func FromString(s string) GUID { + if len(s) != 36 { + panic(fmt.Sprintf("invalid GUID length: %d", len(s))) + } + if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + panic("invalid GUID format") + } + indexOrder := [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34, + } + byteOrder := [16]int{ + 3, 2, 1, 0, + 5, 4, + 7, 6, + 8, 9, + 10, 11, 12, 13, 14, 15, + } + var g GUID + for i, x := range indexOrder { + b, err := strconv.ParseInt(s[x:x+2], 16, 16) + if err != nil { + panic(err) + } + g[byteOrder[i]] = byte(b) + } + return g +} + +func (g GUID) MarshalJSON() ([]byte, error) { + return json.Marshal(g.String()) +} + +func (g *GUID) UnmarshalJSON(data []byte) error { + *g = FromString(strings.Trim(string(data), "\"")) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go index 0de4a706a..8294d66d7 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go @@ -2,6 +2,7 @@ package hcs import ( "encoding/json" + "fmt" "io" "sync" "syscall" @@ -83,7 +84,10 @@ func (process *Process) Kill() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("TerminateProcess %s: %d", process.SystemID(), process.Pid()), &completed) err := hcsTerminateProcess(process.handle, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeProcessError(process, operation, err, events) @@ -177,7 +181,10 @@ func (process *Process) Properties() (*ProcessStatus, error) { resultp *uint16 propertiesp *uint16 ) + completed := false + go syscallWatcher(fmt.Sprintf("GetProcessProperties %s: %d", process.SystemID(), process.Pid()), &completed) err := hcsGetProcessProperties(process.handle, &propertiesp, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeProcessError(process, operation, err, events) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go index 41ff2877b..57afd5ec6 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go @@ -2,6 +2,7 @@ package hcs import ( "encoding/json" + "fmt" "os" "strconv" "sync" @@ -63,7 +64,10 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System, resultp *uint16 identity syscall.Handle ) + completed := false + go syscallWatcher(fmt.Sprintf("CreateCompleteSystem %s: %s", id, hcsDocument), &completed) createError := hcsCreateComputeSystem(id, hcsDocument, identity, &computeSystem.handle, &resultp) + completed = true if createError == nil || IsPending(createError) { if err := computeSystem.registerCallback(); err != nil { @@ -74,7 +78,7 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System, } } - events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.Duration) + events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.SystemCreate) if err != nil { if err == ErrTimeout { // Terminate the compute system if it still exists. We're okay to @@ -135,7 +139,10 @@ func GetComputeSystems(q schema1.ComputeSystemQuery) ([]schema1.ContainerPropert resultp *uint16 computeSystemsp *uint16 ) + completed := false + go syscallWatcher(fmt.Sprintf("GetComputeSystems %s:", query), &completed) err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, &HcsError{Op: operation, Err: err, Events: events} @@ -192,8 +199,11 @@ func (computeSystem *System) Start() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("StartComputeSystem %s:", computeSystem.ID()), &completed) err := hcsStartComputeSystem(computeSystem.handle, "", &resultp) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.Duration) + completed = true + events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart) if err != nil { return makeSystemError(computeSystem, "Start", "", err, events) } @@ -219,7 +229,10 @@ func (computeSystem *System) Shutdown() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("ShutdownComputeSystem %s:", computeSystem.ID()), &completed) err := hcsShutdownComputeSystem(computeSystem.handle, "", &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeSystemError(computeSystem, "Shutdown", "", err, events) @@ -242,7 +255,10 @@ func (computeSystem *System) Terminate() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("TerminateComputeSystem %s:", computeSystem.ID()), &completed) err := hcsTerminateComputeSystem(computeSystem.handle, "", &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeSystemError(computeSystem, "Terminate", "", err, events) @@ -291,7 +307,10 @@ func (computeSystem *System) Properties(types ...schema1.PropertyType) (*schema1 } var resultp, propertiesp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("GetComputeSystemProperties %s:", computeSystem.ID()), &completed) err = hcsGetComputeSystemProperties(computeSystem.handle, string(queryj), &propertiesp, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeSystemError(computeSystem, "Properties", "", err, events) @@ -320,8 +339,11 @@ func (computeSystem *System) Pause() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("PauseComputeSystem %s:", computeSystem.ID()), &completed) err := hcsPauseComputeSystem(computeSystem.handle, "", &resultp) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.Duration) + completed = true + events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause) if err != nil { return makeSystemError(computeSystem, "Pause", "", err, events) } @@ -342,8 +364,11 @@ func (computeSystem *System) Resume() error { } var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("ResumeComputeSystem %s:", computeSystem.ID()), &completed) err := hcsResumeComputeSystem(computeSystem.handle, "", &resultp) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.Duration) + completed = true + events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume) if err != nil { return makeSystemError(computeSystem, "Resume", "", err, events) } @@ -375,7 +400,10 @@ func (computeSystem *System) CreateProcess(c interface{}) (*Process, error) { configuration := string(configurationb) logrus.Debugf(title+" config=%s", configuration) + completed := false + go syscallWatcher(fmt.Sprintf("CreateProcess %s: %s", computeSystem.ID(), configuration), &completed) err = hcsCreateProcess(computeSystem.handle, configuration, &processInfo, &processHandle, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeSystemError(computeSystem, "CreateProcess", configuration, err, events) @@ -415,7 +443,10 @@ func (computeSystem *System) OpenProcess(pid int) (*Process, error) { return nil, makeSystemError(computeSystem, "OpenProcess", "", ErrAlreadyClosed, nil) } + completed := false + go syscallWatcher(fmt.Sprintf("OpenProcess %s: %d", computeSystem.ID(), pid), &completed) err := hcsOpenProcess(computeSystem.handle, uint32(pid), &processHandle, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return nil, makeSystemError(computeSystem, "OpenProcess", "", err, events) @@ -451,7 +482,11 @@ func (computeSystem *System) Close() error { return makeSystemError(computeSystem, "Close", "", err, nil) } - if err := hcsCloseComputeSystem(computeSystem.handle); err != nil { + completed := false + go syscallWatcher(fmt.Sprintf("CloseComputeSystem %s:", computeSystem.ID()), &completed) + err := hcsCloseComputeSystem(computeSystem.handle) + completed = true + if err != nil { return makeSystemError(computeSystem, "Close", "", err, nil) } @@ -537,7 +572,10 @@ func (computeSystem *System) Modify(config interface{}) error { logrus.Debugf(title + " " + requestString) var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("ModifyComputeSystem %s: %s", computeSystem.ID(), requestString), &completed) err = hcsModifyComputeSystem(computeSystem.handle, requestString, &resultp) + completed = true events := processHcsResult(resultp) if err != nil { return makeSystemError(computeSystem, "Modify", requestString, err, events) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go new file mode 100644 index 000000000..6b94bc9ff --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go @@ -0,0 +1,30 @@ +package hcs + +import ( + "time" + + "github.com/Microsoft/hcsshim/internal/timeout" + "github.com/sirupsen/logrus" +) + +// syscallWatcher is used as a very simple goroutine around calls into +// the platform. In some cases, we have seen HCS APIs not returning due to +// various bugs, and the goroutine making the syscall ends up not returning, +// prior to its async callback. By spinning up a syscallWatcher, it allows +// us to at least log a warning if a syscall doesn't complete in a reasonable +// amount of time. +// +// Usage is: +// +// completed := false +// go syscallWatcher("some description", &completed) +// +// completed = true +// +func syscallWatcher(description string, syscallCompleted *bool) { + time.Sleep(timeout.SyscallWatcher) + if *syscallCompleted { + return + } + logrus.Warnf("%s: Did not complete within %s. This may indicate a platform issue. If it appears to be making no forward progress, obtain the stacks and see is there is a syscall stuck in the platform API for a significant length of time.", description, timeout.SyscallWatcher) +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go index 32f4e070c..2f5bf8f55 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go @@ -1,4 +1,4 @@ -// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT +// Code generated by 'go generate'; DO NOT EDIT. package interop diff --git a/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go index 776adbe7a..709b9d347 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go @@ -1,4 +1,4 @@ -// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT +// Code generated by 'go generate'; DO NOT EDIT. package safefile diff --git a/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go b/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go index e4253f400..ff3b6572e 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go @@ -6,21 +6,65 @@ import ( "time" ) -// Duration is the default time to wait for various operations. -// - Waiting for async notifications from HCS -// - Waiting for processes to launch through -// - Waiting to copy data to/from a launched processes stdio pipes. -// -// This can be overridden through environment variable `HCS_TIMEOUT_SECONDS` +var ( + // defaultTimeout is the timeout for most operations that is not overridden. + defaultTimeout = 4 * time.Minute -var Duration = 4 * time.Minute + // defaultTimeoutTestdRetry is the retry loop timeout for testd to respond + // for a disk to come online in LCOW. + defaultTimeoutTestdRetry = 5 * time.Second +) + +// External variables for HCSShim consumers to use. +var ( + // SystemCreate is the timeout for creating a compute system + SystemCreate time.Duration = defaultTimeout + + // SystemStart is the timeout for starting a compute system + SystemStart time.Duration = defaultTimeout + + // SystemPause is the timeout for pausing a compute system + SystemPause time.Duration = defaultTimeout + + // SystemResume is the timeout for resuming a compute system + SystemResume time.Duration = defaultTimeout + + // SyscallWatcher is the timeout before warning of a potential stuck platform syscall. + SyscallWatcher time.Duration = defaultTimeout + + // Tar2VHD is the timeout for the tar2vhd operation to complete + Tar2VHD time.Duration = defaultTimeout + + // ExternalCommandToStart is the timeout for external commands to start + ExternalCommandToStart = defaultTimeout + + // ExternalCommandToComplete is the timeout for external commands to complete. + // Generally this means copying data from their stdio pipes. + ExternalCommandToComplete = defaultTimeout + + // TestDRetryLoop is the timeout for testd retry loop when onlining a SCSI disk in LCOW + TestDRetryLoop = defaultTimeoutTestdRetry +) func init() { - envTimeout := os.Getenv("HCSSHIM_TIMEOUT_SECONDS") + SystemCreate = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMCREATE", SystemCreate) + SystemStart = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMSTART", SystemStart) + SystemPause = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMPAUSE", SystemPause) + SystemResume = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMRESUME", SystemResume) + SyscallWatcher = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSCALLWATCHER", SyscallWatcher) + Tar2VHD = durationFromEnvironment("HCSSHIM_TIMEOUT_TAR2VHD", Tar2VHD) + ExternalCommandToStart = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDSTART", ExternalCommandToStart) + ExternalCommandToComplete = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDCOMPLETE", ExternalCommandToComplete) + TestDRetryLoop = durationFromEnvironment("HCSSHIM_TIMEOUT_TESTDRETRYLOOP", TestDRetryLoop) +} + +func durationFromEnvironment(env string, defaultValue time.Duration) time.Duration { + envTimeout := os.Getenv(env) if len(envTimeout) > 0 { e, err := strconv.Atoi(envTimeout) if err == nil && e > 0 { - Duration = time.Second * time.Duration(e) + return time.Second * time.Duration(e) } } + return defaultValue } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go index a3817843a..d15817730 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go @@ -9,15 +9,15 @@ import ( // the parent layer provided. func CreateLayer(path, parent string) error { title := "hcsshim::CreateLayer " - logrus.Debugf(title+"Flavour %d ID %s parent %s", path, parent) + logrus.Debugf(title+"ID %s parent %s", path, parent) err := createLayer(&stdDriverInfo, path, parent) if err != nil { - err = hcserror.Errorf(err, title, "path=%s parent=%s flavour=%d", path, parent) + err = hcserror.Errorf(err, title, "path=%s parent=%s", path, parent) logrus.Error(err) return err } - logrus.Debugf(title+" - succeeded path=%s parent=%s flavour=%d", path, parent) + logrus.Debugf(title+"- succeeded path=%s parent=%s", path, parent) return nil } diff --git a/vendor/github.com/containerd/containerd/README.md b/vendor/github.com/containerd/containerd/README.md index d3e4aa435..c4d9dffdb 100644 --- a/vendor/github.com/containerd/containerd/README.md +++ b/vendor/github.com/containerd/containerd/README.md @@ -224,7 +224,7 @@ This will be the best place to discuss design and implementation. For sync communication we have a community slack with a #containerd channel that everyone is welcome to join and chat about development. -**Slack:** https://dockr.ly/community +**Slack:** https://join.slack.com/t/dockercommunity/shared_invite/enQtNDM4NjAwNDMyOTUwLWZlMDZmYWRjZjk4Zjc5ZGQ5NWZkOWI1Yjk2NGE3ZWVlYjYxM2VhYjczOWIyZDFhZTE3NTUwZWQzMjhmNGYyZTg ### Reporting security issues diff --git a/vendor/github.com/containerd/containerd/api/services/content/v1/content.pb.go b/vendor/github.com/containerd/containerd/api/services/content/v1/content.pb.go index b539ae1ae..ec08c3b23 100644 --- a/vendor/github.com/containerd/containerd/api/services/content/v1/content.pb.go +++ b/vendor/github.com/containerd/containerd/api/services/content/v1/content.pb.go @@ -443,7 +443,7 @@ type ContentClient interface { // Only one active stream may exist at a time for each ref. // // Once a write stream has started, it may only write to a single ref, thus - // once a stream is started, the ref may be ommitted on subsequent writes. + // once a stream is started, the ref may be omitted on subsequent writes. // // For any write transaction represented by a ref, only a single write may // be made to a given offset. If overlapping writes occur, it is an error. @@ -658,7 +658,7 @@ type ContentServer interface { // Only one active stream may exist at a time for each ref. // // Once a write stream has started, it may only write to a single ref, thus - // once a stream is started, the ref may be ommitted on subsequent writes. + // once a stream is started, the ref may be omitted on subsequent writes. // // For any write transaction represented by a ref, only a single write may // be made to a given offset. If overlapping writes occur, it is an error. diff --git a/vendor/github.com/containerd/containerd/api/services/content/v1/content.proto b/vendor/github.com/containerd/containerd/api/services/content/v1/content.proto index 4f1187145..086b3e39b 100644 --- a/vendor/github.com/containerd/containerd/api/services/content/v1/content.proto +++ b/vendor/github.com/containerd/containerd/api/services/content/v1/content.proto @@ -55,7 +55,7 @@ service Content { // Only one active stream may exist at a time for each ref. // // Once a write stream has started, it may only write to a single ref, thus - // once a stream is started, the ref may be ommitted on subsequent writes. + // once a stream is started, the ref may be omitted on subsequent writes. // // For any write transaction represented by a ref, only a single write may // be made to a given offset. If overlapping writes occur, it is an error. diff --git a/vendor/github.com/containerd/containerd/archive/compression/compression.go b/vendor/github.com/containerd/containerd/archive/compression/compression.go index bd50f083b..bd64e0353 100644 --- a/vendor/github.com/containerd/containerd/archive/compression/compression.go +++ b/vendor/github.com/containerd/containerd/archive/compression/compression.go @@ -20,9 +20,15 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "fmt" "io" + "os" + "os/exec" + "strconv" "sync" + + "github.com/containerd/containerd/log" ) type ( @@ -37,6 +43,13 @@ const ( Gzip ) +const disablePigzEnv = "CONTAINERD_DISABLE_PIGZ" + +var ( + initPigz sync.Once + unpigzPath string +) + var ( bufioReader32KPool = &sync.Pool{ New: func() interface{} { return bufio.NewReaderSize(nil, 32*1024) }, @@ -120,11 +133,18 @@ func DecompressStream(archive io.Reader) (DecompressReadCloser, error) { readBufWrapper := &readCloserWrapper{buf, compression, closer} return readBufWrapper, nil case Gzip: - gzReader, err := gzip.NewReader(buf) + ctx, cancel := context.WithCancel(context.Background()) + gzReader, err := gzipDecompress(ctx, buf) if err != nil { + cancel() return nil, err } - readBufWrapper := &readCloserWrapper{gzReader, compression, closer} + + readBufWrapper := &readCloserWrapper{gzReader, compression, func() error { + cancel() + return closer() + }} + return readBufWrapper, nil default: return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension()) @@ -151,3 +171,67 @@ func (compression *Compression) Extension() string { } return "" } + +func gzipDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) { + initPigz.Do(func() { + if unpigzPath = detectPigz(); unpigzPath != "" { + log.L.Debug("using pigz for decompression") + } + }) + + if unpigzPath == "" { + return gzip.NewReader(buf) + } + + return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf) +} + +func cmdStream(cmd *exec.Cmd, in io.Reader) (io.ReadCloser, error) { + reader, writer := io.Pipe() + + cmd.Stdin = in + cmd.Stdout = writer + + var errBuf bytes.Buffer + cmd.Stderr = &errBuf + + if err := cmd.Start(); err != nil { + return nil, err + } + + go func() { + if err := cmd.Wait(); err != nil { + writer.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) + } else { + writer.Close() + } + }() + + return reader, nil +} + +func detectPigz() string { + path, err := exec.LookPath("unpigz") + if err != nil { + log.L.WithError(err).Debug("unpigz not found, falling back to go gzip") + return "" + } + + // Check if pigz disabled via CONTAINERD_DISABLE_PIGZ env variable + value := os.Getenv(disablePigzEnv) + if value == "" { + return path + } + + disable, err := strconv.ParseBool(value) + if err != nil { + log.L.WithError(err).Warnf("could not parse %s: %s", disablePigzEnv, value) + return path + } + + if disable { + return "" + } + + return path +} diff --git a/vendor/github.com/containerd/containerd/cmd/containerd/command/publish.go b/vendor/github.com/containerd/containerd/cmd/containerd/command/publish.go index e26944b7a..9dabc830a 100644 --- a/vendor/github.com/containerd/containerd/cmd/containerd/command/publish.go +++ b/vendor/github.com/containerd/containerd/cmd/containerd/command/publish.go @@ -99,7 +99,7 @@ func connect(address string, d func(string, time.Duration) (net.Conn, error)) (* grpc.FailOnNonTempDialError(true), grpc.WithBackoffMaxDelay(3 * time.Second), } - ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 60*time.Second) + ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 2*time.Second) defer cancel() conn, err := grpc.DialContext(ctx, dialer.DialAddress(address), gopts...) if err != nil { diff --git a/vendor/github.com/containerd/containerd/cmd/ctr/commands/commands.go b/vendor/github.com/containerd/containerd/cmd/ctr/commands/commands.go index 527516094..cba982454 100644 --- a/vendor/github.com/containerd/containerd/cmd/ctr/commands/commands.go +++ b/vendor/github.com/containerd/containerd/cmd/ctr/commands/commands.go @@ -124,6 +124,10 @@ var ( Name: "gpus", Usage: "add gpus to the container", }, + cli.BoolFlag{ + Name: "allow-new-privs", + Usage: "turn off OCI spec's NoNewPrivileges feature flag", + }, } ) diff --git a/vendor/github.com/containerd/containerd/cmd/ctr/commands/images/import.go b/vendor/github.com/containerd/containerd/cmd/ctr/commands/images/import.go index 37c3ab5c7..2711ad0ee 100644 --- a/vendor/github.com/containerd/containerd/cmd/ctr/commands/images/import.go +++ b/vendor/github.com/containerd/containerd/cmd/ctr/commands/images/import.go @@ -20,10 +20,11 @@ import ( "fmt" "io" "os" + "time" + "github.com/containerd/containerd" "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/images" - oci "github.com/containerd/containerd/images/oci" + "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/log" "github.com/urfave/cli" ) @@ -34,45 +35,58 @@ var importCommand = cli.Command{ ArgsUsage: "[flags] ", Description: `Import images from a tar stream. Implemented formats: -- oci.v1 (default) +- oci.v1 +- docker.v1.1 +- docker.v1.2 -For oci.v1 format, you need to specify --oci-name because an OCI archive contains image refs (tags) -but does not contain the base image name. +For OCI v1, you may need to specify --base-name because an OCI archive may +contain only partial image references (tags without the base image name). +If no base image name is provided, a name will be generated as "import-%{yyyy-MM-dd}". e.g. - $ ctr images import --format oci.v1 --oci-name foo/bar foobar.tar + $ ctr images import --base-name foo/bar foobar.tar If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadbeef", the command will create "foo/bar:latest" and "foo/bar@sha256:deadbeef" images in the containerd store. `, Flags: append([]cli.Flag{ cli.StringFlag{ - Name: "format", - Value: "oci.v1", - Usage: "image format. See DESCRIPTION.", + Name: "base-name", + Value: "", + Usage: "base image name for added images, when provided only images with this name prefix are imported", + }, + cli.BoolFlag{ + Name: "digests", + Usage: "whether to create digest images (default: false)", }, cli.StringFlag{ - Name: "oci-name", - Value: "unknown/unknown", - Usage: "prefix added to either oci.v1 ref annotation or digest", + Name: "index-name", + Usage: "image name to keep index as, by default index is discarded", }, - // TODO(AkihiroSuda): support commands.LabelFlag (for all children objects) }, commands.SnapshotterFlags...), Action: func(context *cli.Context) error { var ( - in = context.Args().First() - imageImporter images.Importer + in = context.Args().First() + opts []containerd.ImportOpt ) - switch format := context.String("format"); format { - case "oci.v1": - imageImporter = &oci.V1Importer{ - ImageName: context.String("oci-name"), - } - default: - return fmt.Errorf("unknown format %s", format) + prefix := context.String("base-name") + if prefix == "" { + prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) + opts = append(opts, containerd.WithImageRefTranslator(archive.AddRefPrefix(prefix))) + } else { + // When provided, filter out references which do not match + opts = append(opts, containerd.WithImageRefTranslator(archive.FilterRefPrefix(prefix))) + } + + if context.Bool("digests") { + opts = append(opts, containerd.WithDigestRef(archive.DigestTranslator(prefix))) + } + + if idxName := context.String("index-name"); idxName != "" { + opts = append(opts, containerd.WithIndexName(idxName)) } client, ctx, cancel, err := commands.NewClient(context) @@ -90,20 +104,24 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb return err } } - imgs, err := client.Import(ctx, imageImporter, r) + imgs, err := client.Import(ctx, r, opts...) + closeErr := r.Close() if err != nil { return err } - if err = r.Close(); err != nil { - return err + if closeErr != nil { + return closeErr } log.G(ctx).Debugf("unpacking %d images", len(imgs)) for _, img := range imgs { + // TODO: Allow configuration of the platform + image := containerd.NewImage(client, img) + // TODO: Show unpack status - fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest) - err = img.Unpack(ctx, context.String("snapshotter")) + fmt.Printf("unpacking %s (%s)...", img.Name, img.Target.Digest) + err = image.Unpack(ctx, context.String("snapshotter")) if err != nil { return err } diff --git a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go index c314b1e8c..314d89177 100644 --- a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go +++ b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go @@ -106,6 +106,10 @@ var Command = cli.Command{ Name: "fifo-dir", Usage: "directory used for storing IO FIFOs", }, + cli.BoolFlag{ + Name: "isolated", + Usage: "run the container with vm isolation", + }, }, append(commands.SnapshotterFlags, commands.ContainerFlags...)...), Action: func(context *cli.Context) error { var ( diff --git a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go index d82b0e648..5f553cd65 100644 --- a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go +++ b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go @@ -136,6 +136,9 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli if context.IsSet("gpus") { opts = append(opts, nvidia.WithGPUs(nvidia.WithDevices(context.Int("gpus")), nvidia.WithAllCapabilities)) } + if context.IsSet("allow-new-privs") { + opts = append(opts, oci.WithNewPrivileges) + } } cOpts = append(cOpts, containerd.WithContainerLabels(commands.LabelArgs(context.StringSlice("label")))) @@ -146,9 +149,8 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli cOpts = append(cOpts, spec) - // oci.WithImageConfig (WithUsername, WithUserID) depends on rootfs snapshot for resolving /etc/passwd. - // So cOpts needs to have precedence over opts. - // TODO: WithUsername, WithUserID should additionally support non-snapshot rootfs + // oci.WithImageConfig (WithUsername, WithUserID) depends on access to rootfs for resolving via + // the /etc/{passwd,group} files. So cOpts needs to have precedence over opts. return client.NewContainer(ctx, id, cOpts...) } diff --git a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_windows.go b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_windows.go index 00d3d7578..ea297aac2 100644 --- a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_windows.go +++ b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_windows.go @@ -31,57 +31,80 @@ import ( // NewContainer creates a new container func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli.Context) (containerd.Container, error) { var ( - ref = context.Args().First() - id = context.Args().Get(1) - args = context.Args()[2:] - ) - - image, err := client.GetImage(ctx, ref) - if err != nil { - return nil, err - } - - var ( + id string opts []oci.SpecOpts cOpts []containerd.NewContainerOpts spec containerd.NewContainerOpts + + config = context.IsSet("config") ) - if context.IsSet("config") { + if config { + id = context.Args().First() opts = append(opts, oci.WithSpecFromFile(context.String("config"))) } else { - opts = append(opts, oci.WithDefaultSpec()) - } + var ( + ref = context.Args().First() + args = context.Args()[2:] + ) - opts = append(opts, oci.WithImageConfig(image)) - opts = append(opts, oci.WithEnv(context.StringSlice("env"))) - opts = append(opts, withMounts(context)) - if context.Bool("tty") { - opts = append(opts, oci.WithTTY) - - con := console.Current() - size, err := con.Size() - if err != nil { - logrus.WithError(err).Error("console size") + id = context.Args().Get(1) + snapshotter := context.String("snapshotter") + if snapshotter == "windows-lcow" { + opts = append(opts, oci.WithDefaultSpecForPlatform("linux/amd64")) + // Clear the rootfs section. + opts = append(opts, oci.WithRootFSPath("")) + } else { + opts = append(opts, oci.WithDefaultSpec()) + } + opts = append(opts, oci.WithEnv(context.StringSlice("env"))) + opts = append(opts, withMounts(context)) + + image, err := client.GetImage(ctx, ref) + if err != nil { + return nil, err + } + unpacked, err := image.IsUnpacked(ctx, snapshotter) + if err != nil { + return nil, err + } + if !unpacked { + if err := image.Unpack(ctx, snapshotter); err != nil { + return nil, err + } + } + opts = append(opts, oci.WithImageConfig(image)) + cOpts = append(cOpts, containerd.WithImage(image)) + cOpts = append(cOpts, containerd.WithSnapshotter(snapshotter)) + cOpts = append(cOpts, containerd.WithNewSnapshot(id, image)) + + if len(args) > 0 { + opts = append(opts, oci.WithProcessArgs(args...)) + } + if cwd := context.String("cwd"); cwd != "" { + opts = append(opts, oci.WithProcessCwd(cwd)) + } + if context.Bool("tty") { + opts = append(opts, oci.WithTTY) + + con := console.Current() + size, err := con.Size() + if err != nil { + logrus.WithError(err).Error("console size") + } + opts = append(opts, oci.WithTTYSize(int(size.Width), int(size.Height))) + } + if context.Bool("isolated") { + opts = append(opts, oci.WithWindowsHyperV) } - opts = append(opts, oci.WithTTYSize(int(size.Width), int(size.Height))) } - if len(args) > 0 { - opts = append(opts, oci.WithProcessArgs(args...)) - } - if cwd := context.String("cwd"); cwd != "" { - opts = append(opts, oci.WithProcessCwd(cwd)) - } + cOpts = append(cOpts, containerd.WithContainerLabels(commands.LabelArgs(context.StringSlice("label")))) + cOpts = append(cOpts, containerd.WithRuntime(context.String("runtime"), nil)) var s specs.Spec spec = containerd.WithSpec(&s, opts...) - cOpts = append(cOpts, containerd.WithContainerLabels(commands.LabelArgs(context.StringSlice("label")))) - cOpts = append(cOpts, containerd.WithImage(image)) - cOpts = append(cOpts, containerd.WithSnapshotter(context.String("snapshotter"))) - cOpts = append(cOpts, containerd.WithNewSnapshot(id, image)) - cOpts = append(cOpts, containerd.WithRuntime(context.String("runtime"), nil)) cOpts = append(cOpts, spec) return client.NewContainer(ctx, id, cOpts...) diff --git a/vendor/github.com/containerd/containerd/containers/containers.go b/vendor/github.com/containerd/containerd/containers/containers.go index e6a562730..a658b5708 100644 --- a/vendor/github.com/containerd/containerd/containers/containers.go +++ b/vendor/github.com/containerd/containerd/containers/containers.go @@ -28,12 +28,12 @@ import ( // // The resources specified in this object are used to create tasks from the container. type Container struct { - // ID uniquely identifies the container in a nameapace. + // ID uniquely identifies the container in a namespace. // // This property is required and cannot be changed after creation. ID string - // Labels provide metadata extension for a contaienr. + // Labels provide metadata extension for a container. // // These are optional and fully mutable. Labels map[string]string diff --git a/vendor/github.com/containerd/containerd/content/helpers.go b/vendor/github.com/containerd/containerd/content/helpers.go index 819b7ea1e..3e231408d 100644 --- a/vendor/github.com/containerd/containerd/content/helpers.go +++ b/vendor/github.com/containerd/containerd/content/helpers.go @@ -70,7 +70,7 @@ func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, desc o cw, err := OpenWriter(ctx, cs, WithRef(ref), WithDescriptor(desc)) if err != nil { if !errdefs.IsAlreadyExists(err) { - return err + return errors.Wrap(err, "failed to open writer") } return nil // all ready present @@ -127,7 +127,7 @@ func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, er func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error { ws, err := cw.Status() if err != nil { - return err + return errors.Wrap(err, "failed to get status") } if ws.Offset > 0 { @@ -138,7 +138,7 @@ func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected dige } if _, err := copyWithBuffer(cw, r); err != nil { - return err + return errors.Wrap(err, "failed to copy") } if err := cw.Commit(ctx, size, expected, opts...); err != nil { diff --git a/vendor/github.com/containerd/containerd/content/local/writer.go b/vendor/github.com/containerd/containerd/content/local/writer.go index a6579a9d2..10df4a4c5 100644 --- a/vendor/github.com/containerd/containerd/content/local/writer.go +++ b/vendor/github.com/containerd/containerd/content/local/writer.go @@ -132,11 +132,11 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, // clean up!! defer os.RemoveAll(w.path) + if _, err := os.Stat(target); err == nil { + // collision with the target file! + return errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", dgst) + } if err := os.Rename(ingest, target); err != nil { - if os.IsExist(err) { - // collision with the target file! - return errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", dgst) - } return err } commitTime := time.Now() diff --git a/vendor/github.com/containerd/containerd/content/proxy/content_writer.go b/vendor/github.com/containerd/containerd/content/proxy/content_writer.go index 6d35ba61a..5434a1568 100644 --- a/vendor/github.com/containerd/containerd/content/proxy/content_writer.go +++ b/vendor/github.com/containerd/containerd/content/proxy/content_writer.go @@ -57,7 +57,7 @@ func (rw *remoteWriter) Status() (content.Status, error) { Action: contentapi.WriteActionStat, }) if err != nil { - return content.Status{}, errors.Wrap(err, "error getting writer status") + return content.Status{}, errors.Wrap(errdefs.FromGRPC(err), "error getting writer status") } return content.Status{ @@ -82,7 +82,7 @@ func (rw *remoteWriter) Write(p []byte) (n int, err error) { Data: p, }) if err != nil { - return 0, err + return 0, errors.Wrap(errdefs.FromGRPC(err), "failed to send write") } n = int(resp.Offset - offset) @@ -112,7 +112,7 @@ func (rw *remoteWriter) Commit(ctx context.Context, size int64, expected digest. Labels: base.Labels, }) if err != nil { - return errdefs.FromGRPC(err) + return errors.Wrap(errdefs.FromGRPC(err), "commit failed") } if size != 0 && resp.Offset != size { diff --git a/vendor/github.com/containerd/containerd/diff/walking/differ.go b/vendor/github.com/containerd/containerd/diff/walking/differ.go index 462d70856..f06aa016f 100644 --- a/vendor/github.com/containerd/containerd/diff/walking/differ.go +++ b/vendor/github.com/containerd/containerd/diff/walking/differ.go @@ -139,7 +139,9 @@ func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, o dgst := cw.Digest() if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil { - return errors.Wrap(err, "failed to commit") + if !errdefs.IsAlreadyExists(err) { + return errors.Wrap(err, "failed to commit") + } } info, err := s.store.Info(ctx, dgst) diff --git a/vendor/github.com/containerd/containerd/export.go b/vendor/github.com/containerd/containerd/export.go index 7aac309ba..bfc25316c 100644 --- a/vendor/github.com/containerd/containerd/export.go +++ b/vendor/github.com/containerd/containerd/export.go @@ -22,6 +22,7 @@ import ( "github.com/containerd/containerd/images" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) type exportOpts struct { @@ -51,7 +52,7 @@ func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocis } pr, pw := io.Pipe() go func() { - pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw)) + pw.CloseWithError(errors.Wrap(exporter.Export(ctx, c.ContentStore(), desc, pw), "export failed")) }() return pr, nil } diff --git a/vendor/github.com/containerd/containerd/image.go b/vendor/github.com/containerd/containerd/image.go index f12cd59c0..62fba9de7 100644 --- a/vendor/github.com/containerd/containerd/image.go +++ b/vendor/github.com/containerd/containerd/image.go @@ -37,6 +37,8 @@ type Image interface { Name() string // Target descriptor for the image content Target() ocispec.Descriptor + // Labels of the image + Labels() map[string]string // Unpack unpacks the image's content into a snapshot Unpack(context.Context, string) error // RootFS returns the unpacked diffids that make up images rootfs. @@ -86,6 +88,10 @@ func (i *image) Target() ocispec.Descriptor { return i.i.Target } +func (i *image) Labels() map[string]string { + return i.i.Labels +} + func (i *image) RootFS(ctx context.Context) ([]digest.Digest, error) { provider := i.client.ContentStore() return i.i.RootFS(ctx, provider, i.platform) diff --git a/vendor/github.com/containerd/containerd/images/archive/importer.go b/vendor/github.com/containerd/containerd/images/archive/importer.go new file mode 100644 index 000000000..59514c66a --- /dev/null +++ b/vendor/github.com/containerd/containerd/images/archive/importer.go @@ -0,0 +1,254 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package archive provides a Docker and OCI compatible importer +package archive + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "io" + "io/ioutil" + "path" + + "github.com/containerd/containerd/archive/compression" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/log" + digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// ImportIndex imports an index from a tar achive image bundle +// - implements Docker v1.1, v1.2 and OCI v1. +// - prefers OCI v1 when provided +// - creates OCI index for Docker formats +// - normalizes Docker references and adds as OCI ref name +// e.g. alpine:latest -> docker.io/library/alpine:latest +// - existing OCI reference names are untouched +// - TODO: support option to compress layers on ingest +func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) { + var ( + tr = tar.NewReader(reader) + + ociLayout ocispec.ImageLayout + mfsts []struct { + Config string + RepoTags []string + Layers []string + } + symlinks = make(map[string]string) + blobs = make(map[string]ocispec.Descriptor) + ) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return ocispec.Descriptor{}, err + } + if hdr.Typeflag == tar.TypeSymlink { + symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname) + } + + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + if hdr.Typeflag != tar.TypeDir { + log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored") + } + continue + } + + hdrName := path.Clean(hdr.Name) + if hdrName == ocispec.ImageLayoutFile { + if err = onUntarJSON(tr, &ociLayout); err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name) + } + } else if hdrName == "manifest.json" { + if err = onUntarJSON(tr, &mfsts); err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name) + } + } else { + dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "failed to ingest %q", hdr.Name) + } + + blobs[hdrName] = ocispec.Descriptor{ + Digest: dgst, + Size: hdr.Size, + } + } + } + + // If OCI layout was given, interpret the tar as an OCI layout. + // When not provided, the layout of the tar will be interpretted + // as Docker v1.1 or v1.2. + if ociLayout.Version != "" { + if ociLayout.Version != ocispec.ImageLayoutVersion { + return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version) + } + + idx, ok := blobs["index.json"] + if !ok { + return ocispec.Descriptor{}, errors.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion) + } + + idx.MediaType = ocispec.MediaTypeImageIndex + return idx, nil + } + + for name, linkname := range symlinks { + desc, ok := blobs[linkname] + if !ok { + return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname) + } + blobs[name] = desc + } + + var idx ocispec.Index + for _, mfst := range mfsts { + config, ok := blobs[mfst.Config] + if !ok { + return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config) + } + config.MediaType = ocispec.MediaTypeImageConfig + + layers, err := resolveLayers(ctx, store, mfst.Layers, blobs) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers") + } + + manifest := ocispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: config, + Layers: layers, + } + + desc, err := writeManifest(ctx, store, manifest, ocispec.MediaTypeImageManifest) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest") + } + + platforms, err := images.Platforms(ctx, store, desc) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform") + } + if len(platforms) > 0 { + // Only one platform can be resolved from non-index manifest, + // The platform can only come from the config included above, + // if the config has no platform it can be safely ommitted. + desc.Platform = &platforms[0] + } + + if len(mfst.RepoTags) == 0 { + idx.Manifests = append(idx.Manifests, desc) + } else { + // Add descriptor per tag + for _, ref := range mfst.RepoTags { + mfstdesc := desc + + normalized, err := normalizeReference(ref) + if err != nil { + return ocispec.Descriptor{}, err + } + + mfstdesc.Annotations = map[string]string{ + ocispec.AnnotationRefName: normalized, + } + + idx.Manifests = append(idx.Manifests, mfstdesc) + } + } + } + + return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex) +} + +func onUntarJSON(r io.Reader, j interface{}) error { + b, err := ioutil.ReadAll(r) + if err != nil { + return err + } + if err := json.Unmarshal(b, j); err != nil { + return err + } + return nil +} + +func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) { + dgstr := digest.Canonical.Digester() + + if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil { + return "", err + } + + return dgstr.Digest(), nil +} + +func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) { + var layers []ocispec.Descriptor + for _, f := range layerFiles { + desc, ok := blobs[f] + if !ok { + return nil, errors.Errorf("layer %q not found", f) + } + + // Open blob, resolve media type + ra, err := store.ReaderAt(ctx, desc) + if err != nil { + return nil, errors.Wrapf(err, "failed to open %q (%s)", f, desc.Digest) + } + s, err := compression.DecompressStream(content.NewReader(ra)) + if err != nil { + return nil, errors.Wrapf(err, "failed to detect compression for %q", f) + } + if s.GetCompression() == compression.Uncompressed { + // TODO: Support compressing and writing back to content store + desc.MediaType = ocispec.MediaTypeImageLayer + } else { + desc.MediaType = ocispec.MediaTypeImageLayerGzip + } + s.Close() + + layers = append(layers, desc) + } + return layers, nil +} + +func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) { + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return ocispec.Descriptor{}, err + } + + desc := ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(manifestBytes), + Size: int64(len(manifestBytes)), + } + if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil { + return ocispec.Descriptor{}, err + } + + return desc, nil +} diff --git a/vendor/github.com/containerd/containerd/images/archive/reference.go b/vendor/github.com/containerd/containerd/images/archive/reference.go new file mode 100644 index 000000000..0b1310181 --- /dev/null +++ b/vendor/github.com/containerd/containerd/images/archive/reference.go @@ -0,0 +1,86 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package archive + +import ( + "strings" + + "github.com/containerd/cri/pkg/util" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// FilterRefPrefix restricts references to having the given image +// prefix. Tag-only references will have the prefix prepended. +func FilterRefPrefix(image string) func(string) string { + return refTranslator(image, true) +} + +// AddRefPrefix prepends the given image prefix to tag-only references, +// while leaving returning full references unmodified. +func AddRefPrefix(image string) func(string) string { + return refTranslator(image, false) +} + +// refTranslator creates a reference which only has a tag or verifies +// a full reference. +func refTranslator(image string, checkPrefix bool) func(string) string { + return func(ref string) string { + // Check if ref is full reference + if strings.ContainsAny(ref, "/:@") { + // If not prefixed, don't include image + if checkPrefix && !isImagePrefix(ref, image) { + return "" + } + return ref + } + return image + ":" + ref + } +} + +func isImagePrefix(s, prefix string) bool { + if !strings.HasPrefix(s, prefix) { + return false + } + if len(s) > len(prefix) { + switch s[len(prefix)] { + case '/', ':', '@': + // Prevent matching partial namespaces + default: + return false + } + } + return true +} + +func normalizeReference(ref string) (string, error) { + // TODO: Replace this function to not depend on reference package + normalized, err := util.NormalizeImageRef(ref) + if err != nil { + return "", errors.Wrapf(err, "normalize image ref %q", ref) + } + + return normalized.String(), nil +} + +// DigestTranslator creates a digest reference by adding the +// digest to an image name +func DigestTranslator(prefix string) func(digest.Digest) string { + return func(dgst digest.Digest) string { + return prefix + "@" + dgst.String() + } +} diff --git a/vendor/github.com/containerd/containerd/images/importexport.go b/vendor/github.com/containerd/containerd/images/importexport.go index 04a55fd38..843adcadc 100644 --- a/vendor/github.com/containerd/containerd/images/importexport.go +++ b/vendor/github.com/containerd/containerd/images/importexport.go @@ -27,7 +27,7 @@ import ( // Importer is the interface for image importer. type Importer interface { // Import imports an image from a tar stream. - Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error) + Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) } // Exporter is the interface for image exporter. diff --git a/vendor/github.com/containerd/containerd/images/oci/exporter.go b/vendor/github.com/containerd/containerd/images/oci/exporter.go index d05c22b64..bf5751e6a 100644 --- a/vendor/github.com/containerd/containerd/images/oci/exporter.go +++ b/vendor/github.com/containerd/containerd/images/oci/exporter.go @@ -94,7 +94,7 @@ func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord { CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { r, err := cs.ReaderAt(ctx, desc) if err != nil { - return 0, err + return 0, errors.Wrap(err, "failed to get reader") } defer r.Close() @@ -103,7 +103,7 @@ func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord { n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) if err != nil { - return 0, err + return 0, errors.Wrap(err, "failed to copy to tar") } if dgstr.Digest() != desc.Digest { return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest()) diff --git a/vendor/github.com/containerd/containerd/images/oci/importer.go b/vendor/github.com/containerd/containerd/images/oci/importer.go deleted file mode 100644 index 64efd3fc2..000000000 --- a/vendor/github.com/containerd/containerd/images/oci/importer.go +++ /dev/null @@ -1,204 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package oci provides the importer and the exporter for OCI Image Spec. -package oci - -import ( - "archive/tar" - "context" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "path" - "strings" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/images" - digest "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" -) - -// V1Importer implements OCI Image Spec v1. -type V1Importer struct { - // ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs). - // This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355 - ImageName string -} - -var _ images.Importer = &V1Importer{} - -// Import implements Importer. -func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) { - if oi.ImageName == "" { - return nil, errors.New("ImageName not set") - } - tr := tar.NewReader(reader) - var imgrecs []images.Image - foundIndexJSON := false - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { - continue - } - hdrName := path.Clean(hdr.Name) - if hdrName == "index.json" { - if foundIndexJSON { - return nil, errors.New("duplicated index.json") - } - foundIndexJSON = true - imgrecs, err = onUntarIndexJSON(tr, oi.ImageName) - if err != nil { - return nil, err - } - continue - } - if strings.HasPrefix(hdrName, "blobs/") { - if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil { - return nil, err - } - } - } - if !foundIndexJSON { - return nil, errors.New("no index.json found") - } - for _, img := range imgrecs { - err := setGCRefContentLabels(ctx, store, img.Target) - if err != nil { - return imgrecs, err - } - } - // FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)? - return imgrecs, nil -} - -func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) { - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - var idx ocispec.Index - if err := json.Unmarshal(b, &idx); err != nil { - return nil, err - } - var imgrecs []images.Image - for _, m := range idx.Manifests { - ref, err := normalizeImageRef(imageName, m) - if err != nil { - return nil, err - } - imgrecs = append(imgrecs, images.Image{ - Name: ref, - Target: m, - }) - } - return imgrecs, nil -} - -func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) { - digest := manifest.Digest - if digest == "" { - return "", errors.Errorf("manifest with empty digest: %v", manifest) - } - ociRef := manifest.Annotations[ocispec.AnnotationRefName] - if ociRef == "" { - return imageName + "@" + digest.String(), nil - } - return imageName + ":" + ociRef, nil -} - -func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error { - // name is like "blobs/sha256/deadbeef" - split := strings.Split(name, "/") - if len(split) != 3 { - return errors.Errorf("unexpected name: %q", name) - } - algo := digest.Algorithm(split[1]) - if !algo.Available() { - return errors.Errorf("unsupported algorithm: %s", algo) - } - dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst}) -} - -// GetChildrenDescriptors returns children blob descriptors for the following supported types: -// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest -// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex -func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - var manifest ocispec.Manifest - if err := json.NewDecoder(r).Decode(&manifest); err != nil { - return nil, err - } - return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil - case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: - var index ocispec.Index - if err := json.NewDecoder(r).Decode(&index); err != nil { - return nil, err - } - return index.Manifests, nil - } - return nil, nil -} - -func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error { - info, err := store.Info(ctx, desc.Digest) - if err != nil { - if errdefs.IsNotFound(err) { - // when the archive is created from multi-arch image, - // it may contain only blobs for a certain platform. - // So ErrNotFound (on manifest list) is expected here. - return nil - } - return err - } - ra, err := store.ReaderAt(ctx, desc) - if err != nil { - return err - } - defer ra.Close() - r := content.NewReader(ra) - children, err := GetChildrenDescriptors(r, desc) - if err != nil { - return err - } - if info.Labels == nil { - info.Labels = map[string]string{} - } - for i, child := range children { - // Note: child blob is not guaranteed to be written to the content store. (multi-arch) - info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String() - } - if _, err := store.Update(ctx, info, "labels"); err != nil { - return err - } - for _, child := range children { - if err := setGCRefContentLabels(ctx, store, child); err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/containerd/containerd/import.go b/vendor/github.com/containerd/containerd/import.go index 7a69f1d45..365056824 100644 --- a/vendor/github.com/containerd/containerd/import.go +++ b/vendor/github.com/containerd/containerd/import.go @@ -18,35 +18,61 @@ package containerd import ( "context" + "encoding/json" "io" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/archive" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) type importOpts struct { + indexName string + imageRefT func(string) string + dgstRefT func(digest.Digest) string } // ImportOpt allows the caller to specify import specific options -type ImportOpt func(c *importOpts) error +type ImportOpt func(*importOpts) error -func resolveImportOpt(opts ...ImportOpt) (importOpts, error) { - var iopts importOpts - for _, o := range opts { - if err := o(&iopts); err != nil { - return iopts, err - } +// WithImageRefTranslator is used to translate the index reference +// to an image reference for the image store. +func WithImageRefTranslator(f func(string) string) ImportOpt { + return func(c *importOpts) error { + c.imageRefT = f + return nil + } +} + +// WithDigestRef is used to create digest images for each +// manifest in the index. +func WithDigestRef(f func(digest.Digest) string) ImportOpt { + return func(c *importOpts) error { + c.dgstRefT = f + return nil + } +} + +// WithIndexName creates a tag pointing to the imported index +func WithIndexName(name string) ImportOpt { + return func(c *importOpts) error { + c.indexName = name + return nil } - return iopts, nil } // Import imports an image from a Tar stream using reader. // Caller needs to specify importer. Future version may use oci.v1 as the default. // Note that unreferrenced blobs may be imported to the content store as well. -func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) { - _, err := resolveImportOpt(opts...) // unused now - if err != nil { - return nil, err +func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt) ([]images.Image, error) { + var iopts importOpts + for _, o := range opts { + if err := o(&iopts); err != nil { + return nil, err + } } ctx, done, err := c.WithLease(ctx) @@ -55,31 +81,86 @@ func (c *Client) Import(ctx context.Context, importer images.Importer, reader io } defer done(ctx) - imgrecs, err := importer.Import(ctx, c.ContentStore(), reader) + index, err := archive.ImportIndex(ctx, c.ContentStore(), reader) if err != nil { - // is.Update() is not called on error return nil, err } - is := c.ImageService() - var images []Image - for _, imgrec := range imgrecs { - if updated, err := is.Update(ctx, imgrec, "target"); err != nil { + var ( + imgs []images.Image + cs = c.ContentStore() + is = c.ImageService() + ) + + if iopts.indexName != "" { + imgs = append(imgs, images.Image{ + Name: iopts.indexName, + Target: index, + }) + } + + var handler images.HandlerFunc + handler = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + // Only save images at top level + if desc.Digest != index.Digest { + return images.Children(ctx, cs, desc) + } + + p, err := content.ReadBlob(ctx, cs, desc) + if err != nil { + return nil, err + } + + var idx ocispec.Index + if err := json.Unmarshal(p, &idx); err != nil { + return nil, err + } + + for _, m := range idx.Manifests { + if ref := m.Annotations[ocispec.AnnotationRefName]; ref != "" { + if iopts.imageRefT != nil { + ref = iopts.imageRefT(ref) + } + if ref != "" { + imgs = append(imgs, images.Image{ + Name: ref, + Target: m, + }) + } + } + if iopts.dgstRefT != nil { + ref := iopts.dgstRefT(m.Digest) + if ref != "" { + imgs = append(imgs, images.Image{ + Name: ref, + Target: m, + }) + } + } + } + + return idx.Manifests, nil + } + + handler = images.SetChildrenLabels(cs, handler) + if err := images.Walk(ctx, handler, index); err != nil { + return nil, err + } + + for i := range imgs { + img, err := is.Update(ctx, imgs[i], "target") + if err != nil { if !errdefs.IsNotFound(err) { return nil, err } - created, err := is.Create(ctx, imgrec) + img, err = is.Create(ctx, imgs[i]) if err != nil { return nil, err } - - imgrec = created - } else { - imgrec = updated } - - images = append(images, NewImage(c, imgrec)) + imgs[i] = img } - return images, nil + + return imgs, nil } diff --git a/vendor/github.com/containerd/containerd/metadata/bolt.go b/vendor/github.com/containerd/containerd/metadata/bolt.go index 156a33593..6ea460866 100644 --- a/vendor/github.com/containerd/containerd/metadata/bolt.go +++ b/vendor/github.com/containerd/containerd/metadata/bolt.go @@ -19,8 +19,8 @@ package metadata import ( "context" - "github.com/boltdb/bolt" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type transactionKey struct{} diff --git a/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go b/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go index 4b2ede876..124018857 100644 --- a/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go +++ b/vendor/github.com/containerd/containerd/metadata/boltutil/helpers.go @@ -19,8 +19,8 @@ package boltutil import ( "time" - "github.com/boltdb/bolt" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) var ( diff --git a/vendor/github.com/containerd/containerd/metadata/buckets.go b/vendor/github.com/containerd/containerd/metadata/buckets.go index d0e1600d1..51d40a7f3 100644 --- a/vendor/github.com/containerd/containerd/metadata/buckets.go +++ b/vendor/github.com/containerd/containerd/metadata/buckets.go @@ -17,8 +17,8 @@ package metadata import ( - "github.com/boltdb/bolt" digest "github.com/opencontainers/go-digest" + bolt "go.etcd.io/bbolt" ) // The layout where a "/" delineates a bucket is described in the following @@ -164,11 +164,11 @@ func getSnapshotterBucket(tx *bolt.Tx, namespace, snapshotter string) *bolt.Buck } func createBlobBucket(tx *bolt.Tx, namespace string, dgst digest.Digest) (*bolt.Bucket, error) { - bkt, err := createBucketIfNotExists(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectContent, bucketKeyObjectBlob, []byte(dgst.String())) + bkt, err := createBucketIfNotExists(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectContent, bucketKeyObjectBlob) if err != nil { return nil, err } - return bkt, nil + return bkt.CreateBucket([]byte(dgst.String())) } func getBlobsBucket(tx *bolt.Tx, namespace string) *bolt.Bucket { diff --git a/vendor/github.com/containerd/containerd/metadata/containers.go b/vendor/github.com/containerd/containerd/metadata/containers.go index 2c9726fc2..6e5622c36 100644 --- a/vendor/github.com/containerd/containerd/metadata/containers.go +++ b/vendor/github.com/containerd/containerd/metadata/containers.go @@ -21,7 +21,6 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" @@ -32,6 +31,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type containerStore struct { diff --git a/vendor/github.com/containerd/containerd/metadata/content.go b/vendor/github.com/containerd/containerd/metadata/content.go index 7293f1bfa..ac4440060 100644 --- a/vendor/github.com/containerd/containerd/metadata/content.go +++ b/vendor/github.com/containerd/containerd/metadata/content.go @@ -23,7 +23,6 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" @@ -34,6 +33,7 @@ import ( digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type contentStore struct { @@ -592,9 +592,6 @@ func (nw *namespacedWriter) commit(ctx context.Context, tx *bolt.Tx, size int64, } size = nw.desc.Size actual = nw.desc.Digest - if getBlobBucket(tx, nw.namespace, actual) != nil { - return "", errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", actual) - } } else { status, err := nw.w.Status() if err != nil { @@ -606,18 +603,16 @@ func (nw *namespacedWriter) commit(ctx context.Context, tx *bolt.Tx, size int64, size = status.Offset actual = nw.w.Digest() - if err := nw.w.Commit(ctx, size, expected); err != nil { - if !errdefs.IsAlreadyExists(err) { - return "", err - } - if getBlobBucket(tx, nw.namespace, actual) != nil { - return "", errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", actual) - } + if err := nw.w.Commit(ctx, size, expected); err != nil && !errdefs.IsAlreadyExists(err) { + return "", err } } bkt, err := createBlobBucket(tx, nw.namespace, actual) if err != nil { + if err == bolt.ErrBucketExists { + return "", errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", actual) + } return "", err } diff --git a/vendor/github.com/containerd/containerd/metadata/db.go b/vendor/github.com/containerd/containerd/metadata/db.go index 0ea8ad78f..507d6d22d 100644 --- a/vendor/github.com/containerd/containerd/metadata/db.go +++ b/vendor/github.com/containerd/containerd/metadata/db.go @@ -23,12 +23,12 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/content" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/log" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) const ( @@ -43,7 +43,7 @@ const ( // dbVersion represents updates to the schema // version which are additions and compatible with // prior version of the same schema. - dbVersion = 2 + dbVersion = 3 ) // DB represents a metadata database backed by a bolt diff --git a/vendor/github.com/containerd/containerd/metadata/gc.go b/vendor/github.com/containerd/containerd/metadata/gc.go index eb4d0c7c3..a670ce1a3 100644 --- a/vendor/github.com/containerd/containerd/metadata/gc.go +++ b/vendor/github.com/containerd/containerd/metadata/gc.go @@ -23,10 +23,10 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/log" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) const ( diff --git a/vendor/github.com/containerd/containerd/metadata/images.go b/vendor/github.com/containerd/containerd/metadata/images.go index 62b1179cd..6ae9dc855 100644 --- a/vendor/github.com/containerd/containerd/metadata/images.go +++ b/vendor/github.com/containerd/containerd/metadata/images.go @@ -23,7 +23,6 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" "github.com/containerd/containerd/images" @@ -33,6 +32,7 @@ import ( digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type imageStore struct { diff --git a/vendor/github.com/containerd/containerd/metadata/leases.go b/vendor/github.com/containerd/containerd/metadata/leases.go index 8835065d1..635c3b3c3 100644 --- a/vendor/github.com/containerd/containerd/metadata/leases.go +++ b/vendor/github.com/containerd/containerd/metadata/leases.go @@ -20,7 +20,6 @@ import ( "context" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/filters" "github.com/containerd/containerd/leases" @@ -28,6 +27,7 @@ import ( "github.com/containerd/containerd/namespaces" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) // LeaseManager manages the create/delete lifecyle of leases diff --git a/vendor/github.com/containerd/containerd/metadata/migrations.go b/vendor/github.com/containerd/containerd/metadata/migrations.go index 3ffda6b3b..34febdd15 100644 --- a/vendor/github.com/containerd/containerd/metadata/migrations.go +++ b/vendor/github.com/containerd/containerd/metadata/migrations.go @@ -16,7 +16,7 @@ package metadata -import "github.com/boltdb/bolt" +import bolt "go.etcd.io/bbolt" type migration struct { schema string @@ -45,6 +45,11 @@ var migrations = []migration{ version: 2, migrate: migrateIngests, }, + { + schema: "v1", + version: 3, + migrate: noOpMigration, + }, } // addChildLinks Adds children key to the snapshotters to enforce snapshot @@ -154,3 +159,10 @@ func migrateIngests(tx *bolt.Tx) error { return nil } + +// noOpMigration was for a database change from boltdb/bolt which is no +// longer being supported, to go.etcd.io/bbolt which is the currently +// maintained repo for boltdb. +func noOpMigration(tx *bolt.Tx) error { + return nil +} diff --git a/vendor/github.com/containerd/containerd/metadata/namespaces.go b/vendor/github.com/containerd/containerd/metadata/namespaces.go index 000bef6c9..74951eb5c 100644 --- a/vendor/github.com/containerd/containerd/metadata/namespaces.go +++ b/vendor/github.com/containerd/containerd/metadata/namespaces.go @@ -19,11 +19,11 @@ package metadata import ( "context" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" l "github.com/containerd/containerd/labels" "github.com/containerd/containerd/namespaces" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type namespaceStore struct { diff --git a/vendor/github.com/containerd/containerd/metadata/snapshot.go b/vendor/github.com/containerd/containerd/metadata/snapshot.go index fda380a6a..75b80b0bc 100644 --- a/vendor/github.com/containerd/containerd/metadata/snapshot.go +++ b/vendor/github.com/containerd/containerd/metadata/snapshot.go @@ -23,7 +23,6 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/labels" "github.com/containerd/containerd/log" @@ -32,6 +31,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) type snapshotter struct { diff --git a/vendor/github.com/containerd/containerd/oci/spec.go b/vendor/github.com/containerd/containerd/oci/spec.go index 6fb31e454..b83f40ac6 100644 --- a/vendor/github.com/containerd/containerd/oci/spec.go +++ b/vendor/github.com/containerd/containerd/oci/spec.go @@ -167,6 +167,7 @@ func populateDefaultUnixSpec(ctx context.Context, s *Spec, id string) error { Destination: "/proc", Type: "proc", Source: "proc", + Options: []string{"nosuid", "noexec", "nodev"}, }, { Destination: "/dev", diff --git a/vendor/github.com/containerd/containerd/oci/spec_opts.go b/vendor/github.com/containerd/containerd/oci/spec_opts.go index d7fe4a29f..ab61d9b26 100644 --- a/vendor/github.com/containerd/containerd/oci/spec_opts.go +++ b/vendor/github.com/containerd/containerd/oci/spec_opts.go @@ -268,6 +268,14 @@ func WithLinuxNamespace(ns specs.LinuxNamespace) SpecOpts { } } +// WithNewPrivileges turns off the NoNewPrivileges feature flag in the spec +func WithNewPrivileges(_ context.Context, _ Client, _ *containers.Container, s *Spec) error { + setProcess(s) + s.Process.NoNewPrivileges = false + + return nil +} + // WithImageConfig configures the spec to from the configuration of an Image func WithImageConfig(image Image) SpecOpts { return WithImageConfigArgs(image, nil) @@ -315,8 +323,14 @@ func WithImageConfigArgs(image Image, args []string) SpecOpts { } s.Process.Cwd = cwd if config.User != "" { - return WithUser(config.User)(ctx, client, c, s) + if err := WithUser(config.User)(ctx, client, c, s); err != nil { + return err + } + return WithAdditionalGIDs(fmt.Sprintf("%d", s.Process.User.UID))(ctx, client, c, s) } + // we should query the image's /etc/group for additional GIDs + // even if there is no specified user in the image config + return WithAdditionalGIDs("root")(ctx, client, c, s) } else if s.Windows != nil { s.Process.Env = config.Env s.Process.Args = append(config.Entrypoint, config.Cmd...) @@ -997,3 +1011,14 @@ var WithPrivileged = Compose( WithApparmorProfile(""), WithSeccompUnconfined, ) + +// WithWindowsHyperV sets the Windows.HyperV section for HyperV isolation of containers. +func WithWindowsHyperV(_ context.Context, _ Client, _ *containers.Container, s *Spec) error { + if s.Windows == nil { + s.Windows = &specs.Windows{} + } + if s.Windows.HyperV == nil { + s.Windows.HyperV = &specs.WindowsHyperV{} + } + return nil +} diff --git a/vendor/github.com/containerd/containerd/runtime/v1/linux/bundle.go b/vendor/github.com/containerd/containerd/runtime/v1/linux/bundle.go index 37a567e28..d73866a2f 100644 --- a/vendor/github.com/containerd/containerd/runtime/v1/linux/bundle.go +++ b/vendor/github.com/containerd/containerd/runtime/v1/linux/bundle.go @@ -46,6 +46,9 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) { return nil, err } path = filepath.Join(path, id) + if err := os.Mkdir(path, 0711); err != nil { + return nil, err + } defer func() { if err != nil { os.RemoveAll(path) @@ -60,10 +63,6 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) { os.RemoveAll(workDir) } }() - - if err := os.Mkdir(path, 0711); err != nil { - return nil, err - } if err := os.Mkdir(filepath.Join(path, "rootfs"), 0711); err != nil { return nil, err } diff --git a/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go b/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go index 24322f07e..d19b8e516 100644 --- a/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go +++ b/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go @@ -26,7 +26,6 @@ import ( "path/filepath" "time" - "github.com/boltdb/bolt" eventstypes "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/containers" @@ -49,6 +48,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" "golang.org/x/sys/unix" ) diff --git a/vendor/github.com/containerd/containerd/runtime/v2/manager.go b/vendor/github.com/containerd/containerd/runtime/v2/manager.go index 3827bd762..a04082dfa 100644 --- a/vendor/github.com/containerd/containerd/runtime/v2/manager.go +++ b/vendor/github.com/containerd/containerd/runtime/v2/manager.go @@ -23,7 +23,6 @@ import ( "os" "path/filepath" - "github.com/boltdb/bolt" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/events/exchange" "github.com/containerd/containerd/log" @@ -32,6 +31,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/runtime" + bolt "go.etcd.io/bbolt" ) func init() { diff --git a/vendor/github.com/containerd/containerd/services/containers/local.go b/vendor/github.com/containerd/containerd/services/containers/local.go index 6f4b3d4d3..95a09872e 100644 --- a/vendor/github.com/containerd/containerd/services/containers/local.go +++ b/vendor/github.com/containerd/containerd/services/containers/local.go @@ -20,7 +20,6 @@ import ( "context" "io" - "github.com/boltdb/bolt" eventstypes "github.com/containerd/containerd/api/events" api "github.com/containerd/containerd/api/services/containers/v1" "github.com/containerd/containerd/containers" @@ -30,6 +29,7 @@ import ( "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/services" ptypes "github.com/gogo/protobuf/types" + bolt "go.etcd.io/bbolt" "google.golang.org/grpc" "google.golang.org/grpc/codes" grpcm "google.golang.org/grpc/metadata" diff --git a/vendor/github.com/containerd/containerd/services/content/service.go b/vendor/github.com/containerd/containerd/services/content/service.go index a27e8ee98..68d3920ff 100644 --- a/vendor/github.com/containerd/containerd/services/content/service.go +++ b/vendor/github.com/containerd/containerd/services/content/service.go @@ -213,7 +213,7 @@ func (s *service) Read(req *api.ReadContentRequest, session api.Content_ReadServ _, err = io.CopyBuffer( &readResponseWriter{session: session}, io.NewSectionReader(ra, offset, size), *p) - return err + return errdefs.ToGRPC(err) } // readResponseWriter is a writer that places the output into ReadContentRequest messages. @@ -420,7 +420,7 @@ func (s *service) Write(session api.Content_WriteServer) (err error) { // maintain the offset as append only, we just issue the write. n, err := wr.Write(req.Data) if err != nil { - return err + return errdefs.ToGRPC(err) } if n != len(req.Data) { @@ -438,7 +438,7 @@ func (s *service) Write(session api.Content_WriteServer) (err error) { opts = append(opts, content.WithLabels(req.Labels)) } if err := wr.Commit(ctx, total, expected, opts...); err != nil { - return err + return errdefs.ToGRPC(err) } } diff --git a/vendor/github.com/containerd/containerd/services/leases/local.go b/vendor/github.com/containerd/containerd/services/leases/local.go index 09aff10f7..0cb310837 100644 --- a/vendor/github.com/containerd/containerd/services/leases/local.go +++ b/vendor/github.com/containerd/containerd/services/leases/local.go @@ -19,12 +19,12 @@ package leases import ( "context" - "github.com/boltdb/bolt" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/metadata" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/services" + bolt "go.etcd.io/bbolt" ) func init() { diff --git a/vendor/github.com/containerd/containerd/services/namespaces/local.go b/vendor/github.com/containerd/containerd/services/namespaces/local.go index 08d21be2b..f50b65355 100644 --- a/vendor/github.com/containerd/containerd/services/namespaces/local.go +++ b/vendor/github.com/containerd/containerd/services/namespaces/local.go @@ -20,7 +20,6 @@ import ( "context" "strings" - "github.com/boltdb/bolt" eventstypes "github.com/containerd/containerd/api/events" api "github.com/containerd/containerd/api/services/namespaces/v1" "github.com/containerd/containerd/errdefs" @@ -30,6 +29,7 @@ import ( "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/services" ptypes "github.com/gogo/protobuf/types" + bolt "go.etcd.io/bbolt" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/vendor/github.com/containerd/containerd/services/server/server.go b/vendor/github.com/containerd/containerd/services/server/server.go index 3b2852e42..ed10766f3 100644 --- a/vendor/github.com/containerd/containerd/services/server/server.go +++ b/vendor/github.com/containerd/containerd/services/server/server.go @@ -29,7 +29,6 @@ import ( "sync" "time" - "github.com/boltdb/bolt" csapi "github.com/containerd/containerd/api/services/content/v1" ssapi "github.com/containerd/containerd/api/services/snapshots/v1" "github.com/containerd/containerd/content" @@ -46,6 +45,7 @@ import ( metrics "github.com/docker/go-metrics" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" "google.golang.org/grpc" ) diff --git a/vendor/github.com/containerd/containerd/services/tasks/local.go b/vendor/github.com/containerd/containerd/services/tasks/local.go index 74e7c1bee..ce9ee59d6 100644 --- a/vendor/github.com/containerd/containerd/services/tasks/local.go +++ b/vendor/github.com/containerd/containerd/services/tasks/local.go @@ -26,7 +26,6 @@ import ( "path/filepath" "time" - "github.com/boltdb/bolt" api "github.com/containerd/containerd/api/services/tasks/v1" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/api/types/task" @@ -48,6 +47,7 @@ import ( ptypes "github.com/gogo/protobuf/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go index 9c589575e..7716a591e 100644 --- a/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go +++ b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go @@ -23,11 +23,11 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/metadata/boltutil" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) var ( diff --git a/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go b/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go index fa5e6c78b..69ba3ea96 100644 --- a/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go +++ b/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go @@ -25,9 +25,9 @@ import ( "context" "sync" - "github.com/boltdb/bolt" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) // Transactor is used to finalize an active transaction. diff --git a/vendor/github.com/containerd/containerd/task.go b/vendor/github.com/containerd/containerd/task.go index 750075f8d..6806e1162 100644 --- a/vendor/github.com/containerd/containerd/task.go +++ b/vendor/github.com/containerd/containerd/task.go @@ -607,8 +607,11 @@ func writeContent(ctx context.Context, store content.Ingester, mediaType, ref st if err != nil { return d, err } + if err := writer.Commit(ctx, size, "", opts...); err != nil { - return d, err + if !errdefs.IsAlreadyExists(err) { + return d, err + } } return v1.Descriptor{ MediaType: mediaType, diff --git a/vendor/github.com/containerd/containerd/vendor.conf b/vendor/github.com/containerd/containerd/vendor.conf index 657cd20ae..4f1229c82 100644 --- a/vendor/github.com/containerd/containerd/vendor.conf +++ b/vendor/github.com/containerd/containerd/vendor.conf @@ -4,7 +4,7 @@ github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/btrfs 2e1aa0ddf94f91fa282b6ed87c23bf0d64911244 -github.com/containerd/continuity f44b615e492bdfb371aae2f76ec694d9da1db537 +github.com/containerd/continuity 7f53d412b9eb1cbf744c2063185d703a0ee34700 github.com/coreos/go-systemd 48702e0da86bd25e76cfef347e2adeb434a0d0a6 github.com/docker/go-metrics 4ea375f7759c82740c893fc030bc37088d2ec098 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 @@ -19,8 +19,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0 github.com/gogo/protobuf v1.0.0 github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef github.com/golang/protobuf v1.1.0 -github.com/opencontainers/runtime-spec d810dbc60d8c5aeeb3d054bd1132fab2121968ce # v1.0.1-43-gd810dbc -github.com/opencontainers/runc 20aff4f0488c6d4b8df4d85b4f63f1f704c11abd +github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d +github.com/opencontainers/runc 00dc70017d222b178a002ed30e9321b12647af2d github.com/sirupsen/logrus v1.0.0 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac @@ -33,18 +33,18 @@ golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 github.com/Microsoft/go-winio v0.4.10 -github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55 -github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd +github.com/Microsoft/hcsshim v0.7.4 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 -github.com/containerd/ttrpc 94dde388801693c54f88a6596f713b51a8b30b2d +github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 gotest.tools v2.1.0 github.com/google/go-cmp v0.1.0 +go.etcd.io/bbolt v1.3.1-etcd.8 # cri dependencies -github.com/containerd/cri v1.11.1 -github.com/containerd/go-cni 5882530828ecf62032409b298a3e8b19e08b6534 +github.com/containerd/cri 9f39e3289533fc228c5e5fcac0a6dbdd60c6047b # release/1.2 branch +github.com/containerd/go-cni 6d7b509a054a3cb1c35ed1865d4fde2f0cb547cd github.com/blang/semver v3.1.0 github.com/containernetworking/cni v0.6.0 github.com/containernetworking/plugins v0.7.0 @@ -52,32 +52,33 @@ github.com/davecgh/go-spew v1.1.0 github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528 -github.com/emicklei/go-restful ff4f55a206334ef123e4f79bbf348980da81ca46 -github.com/ghodss/yaml 73d445a93680fa1a78ae23a5839bad48f32ba1ee +github.com/emicklei/go-restful v2.2.1 +github.com/ghodss/yaml v1.0.0 github.com/golang/glog 44145f04b68cf362d9c4df2182967c2275eaefed github.com/google/gofuzz 44d81051d367757e1c7c6a5a86423ece9afcf63c github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55 github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f -github.com/json-iterator/go f2b4162afba35581b6d4a50d3b8f34e33c144682 -github.com/modern-go/reflect2 05fbef0ca5da472bbf96c9322b84a53edc03c9fd +github.com/json-iterator/go 1.1.5 +github.com/modern-go/reflect2 1.0.1 github.com/modern-go/concurrent 1.0.3 github.com/opencontainers/runtime-tools v0.6.0 -github.com/opencontainers/selinux 4a2974bf1ee960774ffd517717f1f45325af0206 +github.com/opencontainers/selinux b6fa367ed7f534f9ba25391cc2d467085dbb445a github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 -github.com/tchap/go-patricia 5ad6cdb7538b0097d5598c7e57f0a24072adf7dc +github.com/tchap/go-patricia v2.2.6 github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874 golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 +golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 gopkg.in/yaml.v2 v2.2.1 -k8s.io/api 9e5ffd1f1320950b238cfce291b926411f0af722 -k8s.io/apimachinery ed135c5b96450fd24e5e981c708114fbbd950697 -k8s.io/apiserver a90e3a95c2e91b944bfca8225c4e0d12e42a9eb5 -k8s.io/client-go 03bfb9bdcfe5482795b999f39ca3ed9ad42ce5bb -k8s.io/kubernetes v1.11.0 -k8s.io/utils 733eca437aa39379e4bcc25e726439dfca40fcff +k8s.io/api 012f271b5d41baad56190c5f1ae19bff16df0fd8 +k8s.io/apimachinery 6429050ef506887d121f3e7306e894f8900d8a63 +k8s.io/apiserver e9312c15296b6c2c923ebd5031ff5d1d5fd022d7 +k8s.io/client-go 37c3c02ec96533daec0dbda1f39a6b1d68505c79 +k8s.io/kubernetes v1.12.0-beta.1 +k8s.io/utils 982821ea41da7e7c15f3d3738921eb2e7e241ccd # zfs dependencies github.com/containerd/zfs 9a0b8b8b5982014b729cd34eb7cd7a11062aa6ec diff --git a/vendor/github.com/containerd/containerd/version/version.go b/vendor/github.com/containerd/containerd/version/version.go index a34595c7c..f361076ed 100644 --- a/vendor/github.com/containerd/containerd/version/version.go +++ b/vendor/github.com/containerd/containerd/version/version.go @@ -21,7 +21,7 @@ var ( Package = "github.com/containerd/containerd" // Version holds the complete version number. Filled in at linking time. - Version = "1.2.0-beta.2+unknown" + Version = "1.2.0-rc.0+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/github.com/containerd/continuity/context.go b/vendor/github.com/containerd/continuity/context.go index 45b73dc9e..75c98594a 100644 --- a/vendor/github.com/containerd/continuity/context.go +++ b/vendor/github.com/containerd/continuity/context.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/devices/devices.go b/vendor/github.com/containerd/continuity/devices/devices.go index 708640704..e4d4a0370 100644 --- a/vendor/github.com/containerd/continuity/devices/devices.go +++ b/vendor/github.com/containerd/continuity/devices/devices.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package devices import "fmt" diff --git a/vendor/github.com/containerd/continuity/devices/devices_unix.go b/vendor/github.com/containerd/continuity/devices/devices_unix.go index 97fe6b19d..520a5a6f3 100644 --- a/vendor/github.com/containerd/continuity/devices/devices_unix.go +++ b/vendor/github.com/containerd/continuity/devices/devices_unix.go @@ -1,5 +1,21 @@ // +build linux darwin freebsd solaris +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package devices import ( diff --git a/vendor/github.com/containerd/continuity/devices/devices_windows.go b/vendor/github.com/containerd/continuity/devices/devices_windows.go index 6099d1d77..04627c805 100644 --- a/vendor/github.com/containerd/continuity/devices/devices_windows.go +++ b/vendor/github.com/containerd/continuity/devices/devices_windows.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package devices import ( diff --git a/vendor/github.com/containerd/continuity/digests.go b/vendor/github.com/containerd/continuity/digests.go index 355b08039..bf92275db 100644 --- a/vendor/github.com/containerd/continuity/digests.go +++ b/vendor/github.com/containerd/continuity/digests.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/driver/driver.go b/vendor/github.com/containerd/continuity/driver/driver.go index 6a0f76dba..327e96af1 100644 --- a/vendor/github.com/containerd/continuity/driver/driver.go +++ b/vendor/github.com/containerd/continuity/driver/driver.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package driver import ( diff --git a/vendor/github.com/containerd/continuity/driver/driver_unix.go b/vendor/github.com/containerd/continuity/driver/driver_unix.go index c7d4e6ba1..6cb5d10fb 100644 --- a/vendor/github.com/containerd/continuity/driver/driver_unix.go +++ b/vendor/github.com/containerd/continuity/driver/driver_unix.go @@ -1,5 +1,21 @@ // +build linux darwin freebsd solaris +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package driver import ( @@ -13,7 +29,11 @@ import ( ) func (d *driver) Mknod(path string, mode os.FileMode, major, minor int) error { - return devices.Mknod(path, mode, major, minor) + err := devices.Mknod(path, mode, major, minor) + if err != nil { + err = &os.PathError{Op: "mknod", Path: path, Err: err} + } + return err } func (d *driver) Mkfifo(path string, mode os.FileMode) error { @@ -22,7 +42,11 @@ func (d *driver) Mkfifo(path string, mode os.FileMode) error { } // mknod with a mode that has ModeNamedPipe set creates a fifo, not a // device. - return devices.Mknod(path, mode, 0, 0) + err := devices.Mknod(path, mode, 0, 0) + if err != nil { + err = &os.PathError{Op: "mkfifo", Path: path, Err: err} + } + return err } // Getxattr returns all of the extended attributes for the file at path p. diff --git a/vendor/github.com/containerd/continuity/driver/driver_windows.go b/vendor/github.com/containerd/continuity/driver/driver_windows.go index 21c9cf961..f1dcea32a 100644 --- a/vendor/github.com/containerd/continuity/driver/driver_windows.go +++ b/vendor/github.com/containerd/continuity/driver/driver_windows.go @@ -1,18 +1,33 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package driver import ( "os" "github.com/containerd/continuity/sysx" - "github.com/pkg/errors" ) func (d *driver) Mknod(path string, mode os.FileMode, major, minor int) error { - return errors.Wrap(ErrNotSupported, "cannot create device node on Windows") + return &os.PathError{Op: "mknod", Path: path, Err: ErrNotSupported} } func (d *driver) Mkfifo(path string, mode os.FileMode) error { - return errors.Wrap(ErrNotSupported, "cannot create fifo on Windows") + return &os.PathError{Op: "mkfifo", Path: path, Err: ErrNotSupported} } // Lchmod changes the mode of an file not following symlinks. diff --git a/vendor/github.com/containerd/continuity/driver/lchmod_linux.go b/vendor/github.com/containerd/continuity/driver/lchmod_linux.go index 39ffe9cc3..06be28527 100644 --- a/vendor/github.com/containerd/continuity/driver/lchmod_linux.go +++ b/vendor/github.com/containerd/continuity/driver/lchmod_linux.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package driver import ( @@ -15,5 +31,9 @@ func (d *driver) Lchmod(path string, mode os.FileMode) error { return nil } - return unix.Fchmodat(unix.AT_FDCWD, path, uint32(mode), 0) + err := unix.Fchmodat(unix.AT_FDCWD, path, uint32(mode), 0) + if err != nil { + err = &os.PathError{Op: "lchmod", Path: path, Err: err} + } + return err } diff --git a/vendor/github.com/containerd/continuity/driver/lchmod_unix.go b/vendor/github.com/containerd/continuity/driver/lchmod_unix.go index 1b539f78e..b8877a8ae 100644 --- a/vendor/github.com/containerd/continuity/driver/lchmod_unix.go +++ b/vendor/github.com/containerd/continuity/driver/lchmod_unix.go @@ -1,5 +1,21 @@ // +build darwin freebsd solaris +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package driver import ( @@ -10,5 +26,9 @@ import ( // Lchmod changes the mode of a file not following symlinks. func (d *driver) Lchmod(path string, mode os.FileMode) error { - return unix.Fchmodat(unix.AT_FDCWD, path, uint32(mode), unix.AT_SYMLINK_NOFOLLOW) + err := unix.Fchmodat(unix.AT_FDCWD, path, uint32(mode), unix.AT_SYMLINK_NOFOLLOW) + if err != nil { + err = &os.PathError{Op: "lchmod", Path: path, Err: err} + } + return err } diff --git a/vendor/github.com/containerd/continuity/driver/utils.go b/vendor/github.com/containerd/continuity/driver/utils.go index 9e0edd7bc..0c688d158 100644 --- a/vendor/github.com/containerd/continuity/driver/utils.go +++ b/vendor/github.com/containerd/continuity/driver/utils.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package driver import ( diff --git a/vendor/github.com/containerd/continuity/fs/copy.go b/vendor/github.com/containerd/continuity/fs/copy.go index 2ac474b92..42df6a9a5 100644 --- a/vendor/github.com/containerd/continuity/fs/copy.go +++ b/vendor/github.com/containerd/continuity/fs/copy.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/copy_linux.go b/vendor/github.com/containerd/continuity/fs/copy_linux.go index b244e3185..e041b5661 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_linux.go +++ b/vendor/github.com/containerd/continuity/fs/copy_linux.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/copy_unix.go b/vendor/github.com/containerd/continuity/fs/copy_unix.go index 29cbb81ed..1a8ae5ebd 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_unix.go +++ b/vendor/github.com/containerd/continuity/fs/copy_unix.go @@ -1,5 +1,21 @@ // +build solaris darwin freebsd +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/copy_windows.go b/vendor/github.com/containerd/continuity/fs/copy_windows.go index 6fb3de571..be8e6489b 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_windows.go +++ b/vendor/github.com/containerd/continuity/fs/copy_windows.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/diff.go b/vendor/github.com/containerd/continuity/fs/diff.go index f2300e845..e64f9e73d 100644 --- a/vendor/github.com/containerd/continuity/fs/diff.go +++ b/vendor/github.com/containerd/continuity/fs/diff.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/diff_unix.go b/vendor/github.com/containerd/continuity/fs/diff_unix.go index 375181444..7913af27d 100644 --- a/vendor/github.com/containerd/continuity/fs/diff_unix.go +++ b/vendor/github.com/containerd/continuity/fs/diff_unix.go @@ -1,5 +1,21 @@ // +build !windows +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/diff_windows.go b/vendor/github.com/containerd/continuity/fs/diff_windows.go index 8eed36507..4bfa72d3a 100644 --- a/vendor/github.com/containerd/continuity/fs/diff_windows.go +++ b/vendor/github.com/containerd/continuity/fs/diff_windows.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/dtype_linux.go b/vendor/github.com/containerd/continuity/fs/dtype_linux.go index cc06573f1..10510d8de 100644 --- a/vendor/github.com/containerd/continuity/fs/dtype_linux.go +++ b/vendor/github.com/containerd/continuity/fs/dtype_linux.go @@ -1,5 +1,21 @@ // +build linux +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/du.go b/vendor/github.com/containerd/continuity/fs/du.go index f8fc9a994..fccc985dc 100644 --- a/vendor/github.com/containerd/continuity/fs/du.go +++ b/vendor/github.com/containerd/continuity/fs/du.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import "context" diff --git a/vendor/github.com/containerd/continuity/fs/du_unix.go b/vendor/github.com/containerd/continuity/fs/du_unix.go index 9f6bc55fd..e22ffbea3 100644 --- a/vendor/github.com/containerd/continuity/fs/du_unix.go +++ b/vendor/github.com/containerd/continuity/fs/du_unix.go @@ -1,5 +1,21 @@ // +build !windows +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/du_windows.go b/vendor/github.com/containerd/continuity/fs/du_windows.go index faa443fed..8f25ec59c 100644 --- a/vendor/github.com/containerd/continuity/fs/du_windows.go +++ b/vendor/github.com/containerd/continuity/fs/du_windows.go @@ -1,5 +1,21 @@ // +build windows +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/hardlink.go b/vendor/github.com/containerd/continuity/fs/hardlink.go index 38da93813..762aa45e6 100644 --- a/vendor/github.com/containerd/continuity/fs/hardlink.go +++ b/vendor/github.com/containerd/continuity/fs/hardlink.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import "os" diff --git a/vendor/github.com/containerd/continuity/fs/hardlink_unix.go b/vendor/github.com/containerd/continuity/fs/hardlink_unix.go index a6f99778d..f95f0904c 100644 --- a/vendor/github.com/containerd/continuity/fs/hardlink_unix.go +++ b/vendor/github.com/containerd/continuity/fs/hardlink_unix.go @@ -1,5 +1,21 @@ // +build !windows +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/hardlink_windows.go b/vendor/github.com/containerd/continuity/fs/hardlink_windows.go index ad8845a7f..748554714 100644 --- a/vendor/github.com/containerd/continuity/fs/hardlink_windows.go +++ b/vendor/github.com/containerd/continuity/fs/hardlink_windows.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import "os" diff --git a/vendor/github.com/containerd/continuity/fs/path.go b/vendor/github.com/containerd/continuity/fs/path.go index 13fb82638..995981780 100644 --- a/vendor/github.com/containerd/continuity/fs/path.go +++ b/vendor/github.com/containerd/continuity/fs/path.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( @@ -232,12 +248,6 @@ func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, if err != nil { return "", false, err } - if filepath.IsAbs(newpath) && strings.HasPrefix(newpath, root) { - newpath = newpath[:len(root)] - if !strings.HasPrefix(newpath, "/") { - newpath = "/" + newpath - } - } *linksWalked++ return newpath, true, nil } diff --git a/vendor/github.com/containerd/continuity/fs/stat_bsd.go b/vendor/github.com/containerd/continuity/fs/stat_bsd.go index a1b776fdf..cb7400a33 100644 --- a/vendor/github.com/containerd/continuity/fs/stat_bsd.go +++ b/vendor/github.com/containerd/continuity/fs/stat_bsd.go @@ -1,5 +1,21 @@ // +build darwin freebsd +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/stat_linux.go b/vendor/github.com/containerd/continuity/fs/stat_linux.go index 1dbb0212b..4a678dd1f 100644 --- a/vendor/github.com/containerd/continuity/fs/stat_linux.go +++ b/vendor/github.com/containerd/continuity/fs/stat_linux.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import ( diff --git a/vendor/github.com/containerd/continuity/fs/time.go b/vendor/github.com/containerd/continuity/fs/time.go index c336f4d88..cde456123 100644 --- a/vendor/github.com/containerd/continuity/fs/time.go +++ b/vendor/github.com/containerd/continuity/fs/time.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package fs import "time" diff --git a/vendor/github.com/containerd/continuity/groups_unix.go b/vendor/github.com/containerd/continuity/groups_unix.go index e15c14ff8..022d8ab78 100644 --- a/vendor/github.com/containerd/continuity/groups_unix.go +++ b/vendor/github.com/containerd/continuity/groups_unix.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/hardlinks.go b/vendor/github.com/containerd/continuity/hardlinks.go index 8b39bd061..d493dd777 100644 --- a/vendor/github.com/containerd/continuity/hardlinks.go +++ b/vendor/github.com/containerd/continuity/hardlinks.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/hardlinks_unix.go b/vendor/github.com/containerd/continuity/hardlinks_unix.go index 1d81a3f96..a15d1759e 100644 --- a/vendor/github.com/containerd/continuity/hardlinks_unix.go +++ b/vendor/github.com/containerd/continuity/hardlinks_unix.go @@ -1,5 +1,21 @@ // +build linux darwin freebsd solaris +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/hardlinks_windows.go b/vendor/github.com/containerd/continuity/hardlinks_windows.go index be516c560..5893f4e1a 100644 --- a/vendor/github.com/containerd/continuity/hardlinks_windows.go +++ b/vendor/github.com/containerd/continuity/hardlinks_windows.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import "os" diff --git a/vendor/github.com/containerd/continuity/ioutils.go b/vendor/github.com/containerd/continuity/ioutils.go index 3a25bde39..503640ebf 100644 --- a/vendor/github.com/containerd/continuity/ioutils.go +++ b/vendor/github.com/containerd/continuity/ioutils.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/manifest.go b/vendor/github.com/containerd/continuity/manifest.go index f704f048b..8074bbfbb 100644 --- a/vendor/github.com/containerd/continuity/manifest.go +++ b/vendor/github.com/containerd/continuity/manifest.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/pathdriver/path_driver.go b/vendor/github.com/containerd/continuity/pathdriver/path_driver.go index b43d55fe9..b0d5a6b56 100644 --- a/vendor/github.com/containerd/continuity/pathdriver/path_driver.go +++ b/vendor/github.com/containerd/continuity/pathdriver/path_driver.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pathdriver import ( diff --git a/vendor/github.com/containerd/continuity/proto/gen.go b/vendor/github.com/containerd/continuity/proto/gen.go index 8f26ff501..63ce10fb5 100644 --- a/vendor/github.com/containerd/continuity/proto/gen.go +++ b/vendor/github.com/containerd/continuity/proto/gen.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package proto //go:generate protoc --go_out=. manifest.proto diff --git a/vendor/github.com/containerd/continuity/resource.go b/vendor/github.com/containerd/continuity/resource.go index 3643effb3..d2f52bd31 100644 --- a/vendor/github.com/containerd/continuity/resource.go +++ b/vendor/github.com/containerd/continuity/resource.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/resource_unix.go b/vendor/github.com/containerd/continuity/resource_unix.go index 4144643e0..0e103ccc5 100644 --- a/vendor/github.com/containerd/continuity/resource_unix.go +++ b/vendor/github.com/containerd/continuity/resource_unix.go @@ -1,5 +1,21 @@ // +build linux darwin freebsd solaris +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import ( diff --git a/vendor/github.com/containerd/continuity/resource_windows.go b/vendor/github.com/containerd/continuity/resource_windows.go index 7b44414ac..f9801801c 100644 --- a/vendor/github.com/containerd/continuity/resource_windows.go +++ b/vendor/github.com/containerd/continuity/resource_windows.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package continuity import "os" diff --git a/vendor/github.com/containerd/continuity/syscallx/syscall_unix.go b/vendor/github.com/containerd/continuity/syscallx/syscall_unix.go index 4205d1e82..0bfa6a040 100644 --- a/vendor/github.com/containerd/continuity/syscallx/syscall_unix.go +++ b/vendor/github.com/containerd/continuity/syscallx/syscall_unix.go @@ -1,5 +1,21 @@ // +build !windows +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package syscallx import "syscall" diff --git a/vendor/github.com/containerd/continuity/syscallx/syscall_windows.go b/vendor/github.com/containerd/continuity/syscallx/syscall_windows.go index 9637a2875..2ba814990 100644 --- a/vendor/github.com/containerd/continuity/syscallx/syscall_windows.go +++ b/vendor/github.com/containerd/continuity/syscallx/syscall_windows.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package syscallx import ( diff --git a/vendor/github.com/containerd/continuity/sysx/file_posix.go b/vendor/github.com/containerd/continuity/sysx/file_posix.go index d0784f819..e28f3a1b5 100644 --- a/vendor/github.com/containerd/continuity/sysx/file_posix.go +++ b/vendor/github.com/containerd/continuity/sysx/file_posix.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package sysx import ( diff --git a/vendor/github.com/containerd/continuity/sysx/nodata_linux.go b/vendor/github.com/containerd/continuity/sysx/nodata_linux.go index fc47ddb8d..28ce5d8de 100644 --- a/vendor/github.com/containerd/continuity/sysx/nodata_linux.go +++ b/vendor/github.com/containerd/continuity/sysx/nodata_linux.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package sysx import ( diff --git a/vendor/github.com/containerd/continuity/sysx/nodata_solaris.go b/vendor/github.com/containerd/continuity/sysx/nodata_solaris.go index 53cc8e068..e0575f446 100644 --- a/vendor/github.com/containerd/continuity/sysx/nodata_solaris.go +++ b/vendor/github.com/containerd/continuity/sysx/nodata_solaris.go @@ -1,3 +1,19 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package sysx import ( diff --git a/vendor/github.com/containerd/continuity/sysx/nodata_unix.go b/vendor/github.com/containerd/continuity/sysx/nodata_unix.go index 7e6851209..b26f5b3d0 100644 --- a/vendor/github.com/containerd/continuity/sysx/nodata_unix.go +++ b/vendor/github.com/containerd/continuity/sysx/nodata_unix.go @@ -1,5 +1,21 @@ // +build darwin freebsd +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package sysx import ( diff --git a/vendor/github.com/containerd/continuity/sysx/xattr.go b/vendor/github.com/containerd/continuity/sysx/xattr.go index a59efee9a..9e4326dcf 100644 --- a/vendor/github.com/containerd/continuity/sysx/xattr.go +++ b/vendor/github.com/containerd/continuity/sysx/xattr.go @@ -1,5 +1,21 @@ // +build linux darwin +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package sysx import ( diff --git a/vendor/github.com/containerd/continuity/sysx/xattr_unsupported.go b/vendor/github.com/containerd/continuity/sysx/xattr_unsupported.go index 4f6a12e35..c9ef3a1d2 100644 --- a/vendor/github.com/containerd/continuity/sysx/xattr_unsupported.go +++ b/vendor/github.com/containerd/continuity/sysx/xattr_unsupported.go @@ -1,5 +1,21 @@ // +build !linux,!darwin +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package sysx import ( diff --git a/vendor/github.com/containerd/ttrpc/client.go b/vendor/github.com/containerd/ttrpc/client.go index ede2e6b74..e40592dd7 100644 --- a/vendor/github.com/containerd/ttrpc/client.go +++ b/vendor/github.com/containerd/ttrpc/client.go @@ -110,12 +110,16 @@ func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) err } select { + case <-ctx.Done(): + return ctx.Err() case c.calls <- call: case <-c.done: return c.err } select { + case <-ctx.Done(): + return ctx.Err() case err := <-errs: return filterCloseErr(err) case <-c.done: diff --git a/vendor/github.com/containerd/ttrpc/server.go b/vendor/github.com/containerd/ttrpc/server.go index 1a518b1b4..263cb4583 100644 --- a/vendor/github.com/containerd/ttrpc/server.go +++ b/vendor/github.com/containerd/ttrpc/server.go @@ -127,13 +127,13 @@ func (s *Server) Serve(ctx context.Context, l net.Listener) error { func (s *Server) Shutdown(ctx context.Context) error { s.mu.Lock() - lnerr := s.closeListeners() select { case <-s.done: default: // protected by mutex close(s.done) } + lnerr := s.closeListeners() s.mu.Unlock() ticker := time.NewTicker(200 * time.Millisecond) diff --git a/vendor/github.com/opencontainers/runc/README.md b/vendor/github.com/opencontainers/runc/README.md index 5215e32c1..83379d962 100644 --- a/vendor/github.com/opencontainers/runc/README.md +++ b/vendor/github.com/opencontainers/runc/README.md @@ -87,6 +87,18 @@ You can run a specific test case by setting the `TESTFLAGS` variable. # make test TESTFLAGS="-run=SomeTestFunction" ``` +You can run a specific integration test by setting the `TESTPATH` variable. + +```bash +# make test TESTPATH="/checkpoint.bats" +``` + +You can run a test in your proxy environment by setting `DOCKER_BUILD_PROXY` and `DOCKER_RUN_PROXY` variables. + +```bash +# make test DOCKER_BUILD_PROXY="--build-arg HTTP_PROXY=http://yourproxy/" DOCKER_RUN_PROXY="-e HTTP_PROXY=http://yourproxy/" +``` + ### Dependencies Management `runc` uses [vndr](https://github.com/LK4D4/vndr) for dependencies management. diff --git a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md index 575701371..9ec6c3931 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md +++ b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md @@ -10,8 +10,8 @@ The `nsenter` package will `import "C"` and it uses [cgo](https://golang.org/cmd package. In cgo, if the import of "C" is immediately preceded by a comment, that comment, called the preamble, is used as a header when compiling the C parts of the package. So every time we import package `nsenter`, the C code function `nsexec()` would be -called. And package `nsenter` is now only imported in `main_unix.go`, so every time -before we call `cmd.Start` on linux, that C code would run. +called. And package `nsenter` is only imported in `init.go`, so every time the runc +`init` command is invoked, that C code is run. Because `nsexec()` must be run before the Go runtime in order to use the Linux kernel namespace, you must `import` this library into a package if @@ -37,7 +37,7 @@ the parent `nsexec()` will exit and the child `nsexec()` process will return to allow the Go runtime take over. NOTE: We do both `setns(2)` and `clone(2)` even if we don't have any -CLONE_NEW* clone flags because we must fork a new process in order to +`CLONE_NEW*` clone flags because we must fork a new process in order to enter the PID namespace. diff --git a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index a4cd1399d..cb2243147 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -211,7 +211,7 @@ static int try_mapping_tool(const char *app, int pid, char *map, size_t map_len) /* * If @app is NULL, execve will segfault. Just check it here and bail (if - * we're in this path, the caller is already getting desparate and there + * we're in this path, the caller is already getting desperate and there * isn't a backup to this failing). This usually would be a configuration * or programming issue. */ diff --git a/vendor/github.com/boltdb/bolt/LICENSE b/vendor/go.etcd.io/bbolt/LICENSE similarity index 100% rename from vendor/github.com/boltdb/bolt/LICENSE rename to vendor/go.etcd.io/bbolt/LICENSE diff --git a/vendor/github.com/boltdb/bolt/README.md b/vendor/go.etcd.io/bbolt/README.md similarity index 90% rename from vendor/github.com/boltdb/bolt/README.md rename to vendor/go.etcd.io/bbolt/README.md index 7d43a15b2..6546fc01a 100644 --- a/vendor/github.com/boltdb/bolt/README.md +++ b/vendor/go.etcd.io/bbolt/README.md @@ -1,5 +1,18 @@ -Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg) -==== +bbolt +===== + +[![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/bbolt) +[![Coverage](https://codecov.io/gh/etcd-io/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/etcd-io/bbolt) +[![Build Status Travis](https://img.shields.io/travis/etcd-io/bboltlabs.svg?style=flat-square&&branch=master)](https://travis-ci.com/etcd-io/bbolt) +[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/etcd-io/bbolt) +[![Releases](https://img.shields.io/github/release/etcd-io/bbolt/all.svg?style=flat-square)](https://github.com/etcd-io/bbolt/releases) +[![LICENSE](https://img.shields.io/github/license/etcd-io/bbolt.svg?style=flat-square)](https://github.com/etcd-io/bbolt/blob/master/LICENSE) + +bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value +store. The purpose of this fork is to provide the Go community with an active +maintenance and development target for Bolt; the goal is improved reliability +and stability. bbolt includes bug fixes, performance enhancements, and features +not found in Bolt while preserving backwards compatibility with the Bolt API. Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] [LMDB project][lmdb]. The goal of the project is to provide a simple, @@ -10,6 +23,8 @@ Since Bolt is meant to be used as such a low-level piece of functionality, simplicity is key. The API will be small and only focus on getting values and setting values. That's it. +[gh_ben]: https://github.com/benbjohnson +[bolt]: https://github.com/boltdb/bolt [hyc_symas]: https://twitter.com/hyc_symas [lmdb]: http://symas.com/mdb/ @@ -21,36 +36,42 @@ consistency and thread safety. Bolt is currently used in high-load production environments serving databases as large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed services every day. +## Project versioning + +bbolt uses [semantic versioning](http://semver.org). +API should not change between patch and minor releases. +New minor versions may add additional features to the API. + ## Table of Contents -- [Getting Started](#getting-started) - - [Installing](#installing) - - [Opening a database](#opening-a-database) - - [Transactions](#transactions) - - [Read-write transactions](#read-write-transactions) - - [Read-only transactions](#read-only-transactions) - - [Batch read-write transactions](#batch-read-write-transactions) - - [Managing transactions manually](#managing-transactions-manually) - - [Using buckets](#using-buckets) - - [Using key/value pairs](#using-keyvalue-pairs) - - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) - - [Iterating over keys](#iterating-over-keys) - - [Prefix scans](#prefix-scans) - - [Range scans](#range-scans) - - [ForEach()](#foreach) - - [Nested buckets](#nested-buckets) - - [Database backups](#database-backups) - - [Statistics](#statistics) - - [Read-Only Mode](#read-only-mode) - - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) -- [Resources](#resources) -- [Comparison with other databases](#comparison-with-other-databases) - - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) - - [LevelDB, RocksDB](#leveldb-rocksdb) - - [LMDB](#lmdb) -- [Caveats & Limitations](#caveats--limitations) -- [Reading the Source](#reading-the-source) -- [Other Projects Using Bolt](#other-projects-using-bolt) + - [Getting Started](#getting-started) + - [Installing](#installing) + - [Opening a database](#opening-a-database) + - [Transactions](#transactions) + - [Read-write transactions](#read-write-transactions) + - [Read-only transactions](#read-only-transactions) + - [Batch read-write transactions](#batch-read-write-transactions) + - [Managing transactions manually](#managing-transactions-manually) + - [Using buckets](#using-buckets) + - [Using key/value pairs](#using-keyvalue-pairs) + - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) + - [Iterating over keys](#iterating-over-keys) + - [Prefix scans](#prefix-scans) + - [Range scans](#range-scans) + - [ForEach()](#foreach) + - [Nested buckets](#nested-buckets) + - [Database backups](#database-backups) + - [Statistics](#statistics) + - [Read-Only Mode](#read-only-mode) + - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) + - [Resources](#resources) + - [Comparison with other databases](#comparison-with-other-databases) + - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) + - [LevelDB, RocksDB](#leveldb-rocksdb) + - [LMDB](#lmdb) + - [Caveats & Limitations](#caveats--limitations) + - [Reading the Source](#reading-the-source) + - [Other Projects Using Bolt](#other-projects-using-bolt) ## Getting Started @@ -59,13 +80,28 @@ Shopify and Heroku use Bolt-backed services every day. To start using Bolt, install Go and run `go get`: ```sh -$ go get github.com/boltdb/bolt/... +$ go get go.etcd.io/bbolt/... ``` This will retrieve the library and install the `bolt` command line utility into your `$GOBIN` path. +### Importing bbolt + +To use bbolt as an embedded key-value store, import as: + +```go +import bolt "go.etcd.io/bbolt" + +db, err := bolt.Open(path, 0666, nil) +if err != nil { + return err +} +defer db.Close() +``` + + ### Opening a database The top-level object in Bolt is a `DB`. It is represented as a single file on @@ -79,7 +115,7 @@ package main import ( "log" - "github.com/boltdb/bolt" + bolt "go.etcd.io/bbolt" ) func main() { @@ -522,7 +558,7 @@ this from a read-only transaction, it will perform a hot backup and not block your other database reads and writes. By default, it will use a regular file handle which will utilize the operating -system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx) +system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx) documentation for information about optimizing for larger-than-RAM datasets. One common use case is to backup over HTTP so you can use tools like `cURL` to @@ -811,7 +847,7 @@ Here are a few things to note when evaluating and using Bolt: ## Reading the Source -Bolt is a relatively small code base (<3KLOC) for an embedded, serializable, +Bolt is a relatively small code base (<5KLOC) for an embedded, serializable, transactional key/value database so it can be a good starting point for people interested in how databases work. @@ -863,54 +899,55 @@ them via pull request. Below is a list of public, open source projects that use Bolt: -* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. -* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. -* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. -* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. -* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. -* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. -* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations. -* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. -* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". -* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka. -* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed. +* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. +* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support. +* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB * [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. -* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. -* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage. -* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. -* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. +* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. -* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. -* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. -* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. -* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. -* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. -* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. -* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. -* [stow](https://github.com/djherbis/stow) - a persistence manager for objects - backed by boltdb. +* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. * [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining simple tx and key scans. -* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. -* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service -* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. -* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. -* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. -* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. -* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. -* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. -* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. -* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. -* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. -* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. -* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains -* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. -* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. +* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. +* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations. +* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. +* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. * [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. +* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. +* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka. +* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. +* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains +* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. +* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. +* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed. * [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies -* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB -* [Ponzu CMS](https://ponzu-cms.org) - Headless CMS + automatic JSON API with auto-HTTPS, HTTP/2 Server Push, and flexible server framework. +* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. +* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage. +* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. +* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. +* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. +* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. +* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi. +* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service +* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. +* [stow](https://github.com/djherbis/stow) - a persistence manager for objects + backed by boltdb. +* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. +* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. +* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. +* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. +* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. +* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. +* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/boltdb/bolt/bolt_386.go b/vendor/go.etcd.io/bbolt/bolt_386.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_386.go rename to vendor/go.etcd.io/bbolt/bolt_386.go index 820d533c1..4d35ee7cf 100644 --- a/vendor/github.com/boltdb/bolt/bolt_386.go +++ b/vendor/go.etcd.io/bbolt/bolt_386.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/boltdb/bolt/bolt_amd64.go b/vendor/go.etcd.io/bbolt/bolt_amd64.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_amd64.go rename to vendor/go.etcd.io/bbolt/bolt_amd64.go index 98fafdb47..60a52dad5 100644 --- a/vendor/github.com/boltdb/bolt/bolt_amd64.go +++ b/vendor/go.etcd.io/bbolt/bolt_amd64.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_arm.go b/vendor/go.etcd.io/bbolt/bolt_arm.go similarity index 98% rename from vendor/github.com/boltdb/bolt/bolt_arm.go rename to vendor/go.etcd.io/bbolt/bolt_arm.go index 7e5cb4b94..105d27ddb 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm.go +++ b/vendor/go.etcd.io/bbolt/bolt_arm.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "unsafe" diff --git a/vendor/github.com/boltdb/bolt/bolt_arm64.go b/vendor/go.etcd.io/bbolt/bolt_arm64.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_arm64.go rename to vendor/go.etcd.io/bbolt/bolt_arm64.go index b26d84f91..f5aa2a5ee 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm64.go +++ b/vendor/go.etcd.io/bbolt/bolt_arm64.go @@ -1,6 +1,6 @@ // +build arm64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_linux.go b/vendor/go.etcd.io/bbolt/bolt_linux.go similarity index 91% rename from vendor/github.com/boltdb/bolt/bolt_linux.go rename to vendor/go.etcd.io/bbolt/bolt_linux.go index 2b6766614..7707bcacf 100644 --- a/vendor/github.com/boltdb/bolt/bolt_linux.go +++ b/vendor/go.etcd.io/bbolt/bolt_linux.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/go.etcd.io/bbolt/bolt_mips64x.go b/vendor/go.etcd.io/bbolt/bolt_mips64x.go new file mode 100644 index 000000000..baeb289fd --- /dev/null +++ b/vendor/go.etcd.io/bbolt/bolt_mips64x.go @@ -0,0 +1,12 @@ +// +build mips64 mips64le + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x8000000000 // 512GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/go.etcd.io/bbolt/bolt_mipsx.go b/vendor/go.etcd.io/bbolt/bolt_mipsx.go new file mode 100644 index 000000000..2d9b1a91f --- /dev/null +++ b/vendor/go.etcd.io/bbolt/bolt_mipsx.go @@ -0,0 +1,12 @@ +// +build mips mipsle + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x40000000 // 1GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_openbsd.go b/vendor/go.etcd.io/bbolt/bolt_openbsd.go similarity index 97% rename from vendor/github.com/boltdb/bolt/bolt_openbsd.go rename to vendor/go.etcd.io/bbolt/bolt_openbsd.go index 7058c3d73..d7f50358e 100644 --- a/vendor/github.com/boltdb/bolt/bolt_openbsd.go +++ b/vendor/go.etcd.io/bbolt/bolt_openbsd.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc.go b/vendor/go.etcd.io/bbolt/bolt_ppc.go similarity index 69% rename from vendor/github.com/boltdb/bolt/bolt_ppc.go rename to vendor/go.etcd.io/bbolt/bolt_ppc.go index 645ddc3ed..69804714a 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc.go @@ -1,9 +1,12 @@ // +build ppc -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64.go b/vendor/go.etcd.io/bbolt/bolt_ppc64.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_ppc64.go rename to vendor/go.etcd.io/bbolt/bolt_ppc64.go index 9331d9771..356590857 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc64.go @@ -1,6 +1,6 @@ // +build ppc64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go b/vendor/go.etcd.io/bbolt/bolt_ppc64le.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_ppc64le.go rename to vendor/go.etcd.io/bbolt/bolt_ppc64le.go index 8c143bc5d..422c7c69d 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go +++ b/vendor/go.etcd.io/bbolt/bolt_ppc64le.go @@ -1,6 +1,6 @@ // +build ppc64le -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_s390x.go b/vendor/go.etcd.io/bbolt/bolt_s390x.go similarity index 95% rename from vendor/github.com/boltdb/bolt/bolt_s390x.go rename to vendor/go.etcd.io/bbolt/bolt_s390x.go index d7c39af92..6d3fcb825 100644 --- a/vendor/github.com/boltdb/bolt/bolt_s390x.go +++ b/vendor/go.etcd.io/bbolt/bolt_s390x.go @@ -1,6 +1,6 @@ // +build s390x -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_unix.go b/vendor/go.etcd.io/bbolt/bolt_unix.go similarity index 71% rename from vendor/github.com/boltdb/bolt/bolt_unix.go rename to vendor/go.etcd.io/bbolt/bolt_unix.go index cad62dda1..5f2bb5145 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix.go +++ b/vendor/go.etcd.io/bbolt/bolt_unix.go @@ -1,41 +1,43 @@ // +build !windows,!plan9,!solaris -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + flag := syscall.LOCK_NB + if exclusive { + flag |= syscall.LOCK_EX + } else { + flag |= syscall.LOCK_SH + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - flag := syscall.LOCK_SH - if exclusive { - flag = syscall.LOCK_EX - } - - // Otherwise attempt to obtain an exclusive lock. - err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB) + // Attempt to obtain an exclusive lock. + err := syscall.Flock(int(fd), flag) if err == nil { return nil } else if err != syscall.EWOULDBLOCK { return err } + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } @@ -53,7 +55,9 @@ func mmap(db *DB, sz int) error { } // Advise the kernel that the mmap is accessed randomly. - if err := madvise(b, syscall.MADV_RANDOM); err != nil { + err = madvise(b, syscall.MADV_RANDOM) + if err != nil && err != syscall.ENOSYS { + // Ignore not implemented error in kernel because it still works. return fmt.Errorf("madvise: %s", err) } diff --git a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go b/vendor/go.etcd.io/bbolt/bolt_unix_solaris.go similarity index 70% rename from vendor/github.com/boltdb/bolt/bolt_unix_solaris.go rename to vendor/go.etcd.io/bbolt/bolt_unix_solaris.go index 307bf2b3e..babad6578 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go +++ b/vendor/go.etcd.io/bbolt/bolt_unix_solaris.go @@ -1,8 +1,7 @@ -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" @@ -11,36 +10,35 @@ import ( ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + var lockType int16 + if exclusive { + lockType = syscall.F_WRLCK + } else { + lockType = syscall.F_RDLCK + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - var lock syscall.Flock_t - lock.Start = 0 - lock.Len = 0 - lock.Pid = 0 - lock.Whence = 0 - lock.Pid = 0 - if exclusive { - lock.Type = syscall.F_WRLCK - } else { - lock.Type = syscall.F_RDLCK - } - err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock) + // Attempt to obtain an exclusive lock. + lock := syscall.Flock_t{Type: lockType} + err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) if err == nil { return nil } else if err != syscall.EAGAIN { return err } + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } diff --git a/vendor/github.com/boltdb/bolt/bolt_windows.go b/vendor/go.etcd.io/bbolt/bolt_windows.go similarity index 76% rename from vendor/github.com/boltdb/bolt/bolt_windows.go rename to vendor/go.etcd.io/bbolt/bolt_windows.go index b00fb0720..fca178bd2 100644 --- a/vendor/github.com/boltdb/bolt/bolt_windows.go +++ b/vendor/go.etcd.io/bbolt/bolt_windows.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -16,8 +16,6 @@ var ( ) const ( - lockExt = ".lock" - // see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx flagLockExclusive = 2 flagLockFailImmediately = 1 @@ -48,48 +46,47 @@ func fdatasync(db *DB) error { } // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { - // Create a separate lock file on windows because a process - // cannot share an exclusive lock on the same file. This is - // needed during Tx.WriteTo(). - f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode) - if err != nil { - return err - } - db.lockfile = f - +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + var flag uint32 = flagLockFailImmediately + if exclusive { + flag |= flagLockExclusive + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } + // Fix for https://github.com/etcd-io/bbolt/issues/121. Use byte-range + // -1..0 as the lock on the database file. + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := lockFileEx(syscall.Handle(db.file.Fd()), flag, 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) - var flag uint32 = flagLockFailImmediately - if exclusive { - flag |= flagLockExclusive - } - - err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{}) if err == nil { return nil } else if err != errLockViolation { return err } + // If we timed oumercit then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } // funlock releases an advisory lock on a file descriptor. func funlock(db *DB) error { - err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{}) - db.lockfile.Close() - os.Remove(db.path + lockExt) + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := unlockFileEx(syscall.Handle(db.file.Fd()), 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) return err } diff --git a/vendor/github.com/boltdb/bolt/boltsync_unix.go b/vendor/go.etcd.io/bbolt/boltsync_unix.go similarity index 91% rename from vendor/github.com/boltdb/bolt/boltsync_unix.go rename to vendor/go.etcd.io/bbolt/boltsync_unix.go index f50442523..9587afefe 100644 --- a/vendor/github.com/boltdb/bolt/boltsync_unix.go +++ b/vendor/go.etcd.io/bbolt/boltsync_unix.go @@ -1,6 +1,6 @@ // +build !windows,!plan9,!linux,!openbsd -package bolt +package bbolt // fdatasync flushes written data to a file descriptor. func fdatasync(db *DB) error { diff --git a/vendor/github.com/boltdb/bolt/bucket.go b/vendor/go.etcd.io/bbolt/bucket.go similarity index 99% rename from vendor/github.com/boltdb/bolt/bucket.go rename to vendor/go.etcd.io/bbolt/bucket.go index 0c5bf2746..84bfd4d6a 100644 --- a/vendor/github.com/boltdb/bolt/bucket.go +++ b/vendor/go.etcd.io/bbolt/bucket.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -14,13 +14,6 @@ const ( MaxValueSize = (1 << 31) - 2 ) -const ( - maxUint = ^uint(0) - minUint = 0 - maxInt = int(^uint(0) >> 1) - minInt = -maxInt - 1 -) - const bucketHeaderSize = int(unsafe.Sizeof(bucket{})) const ( @@ -323,7 +316,12 @@ func (b *Bucket) Delete(key []byte) error { // Move cursor to correct position. c := b.Cursor() - _, _, flags := c.seek(key) + k, _, flags := c.seek(key) + + // Return nil if the key doesn't exist. + if !bytes.Equal(key, k) { + return nil + } // Return an error if there is already existing bucket value. if (flags & bucketLeafFlag) != 0 { diff --git a/vendor/github.com/boltdb/bolt/cursor.go b/vendor/go.etcd.io/bbolt/cursor.go similarity index 99% rename from vendor/github.com/boltdb/bolt/cursor.go rename to vendor/go.etcd.io/bbolt/cursor.go index 1be9f35e3..3000aced6 100644 --- a/vendor/github.com/boltdb/bolt/cursor.go +++ b/vendor/go.etcd.io/bbolt/cursor.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -157,12 +157,6 @@ func (c *Cursor) seek(seek []byte) (key []byte, value []byte, flags uint32) { // Start from root page/node and traverse to correct page. c.stack = c.stack[:0] c.search(seek, c.bucket.root) - ref := &c.stack[len(c.stack)-1] - - // If the cursor is pointing to the end of page/node then return nil. - if ref.index >= ref.count() { - return nil, nil, 0 - } // If this is a bucket then return a nil value. return c.keyValue() @@ -339,6 +333,8 @@ func (c *Cursor) nsearch(key []byte) { // keyValue returns the key and value of the current leaf element. func (c *Cursor) keyValue() ([]byte, []byte, uint32) { ref := &c.stack[len(c.stack)-1] + + // If the cursor is pointing to the end of page/node then return nil. if ref.count() == 0 || ref.index >= ref.count() { return nil, nil, 0 } diff --git a/vendor/github.com/boltdb/bolt/db.go b/vendor/go.etcd.io/bbolt/db.go similarity index 84% rename from vendor/github.com/boltdb/bolt/db.go rename to vendor/go.etcd.io/bbolt/db.go index f352ff14f..d91dcf813 100644 --- a/vendor/github.com/boltdb/bolt/db.go +++ b/vendor/go.etcd.io/bbolt/db.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "errors" @@ -7,8 +7,7 @@ import ( "log" "os" "runtime" - "runtime/debug" - "strings" + "sort" "sync" "time" "unsafe" @@ -23,6 +22,8 @@ const version = 2 // Represents a marker value to indicate that a file is a Bolt DB. const magic uint32 = 0xED0CDAED +const pgidNoFreelist pgid = 0xffffffffffffffff + // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when // syncing changes to a file. This is required as some operating systems, // such as OpenBSD, do not have a unified buffer cache (UBC) and writes @@ -39,6 +40,9 @@ const ( // default page size for db is set to the OS page size. var defaultPageSize = os.Getpagesize() +// The time elapsed between consecutive file locking attempts. +const flockRetryTimeout = 50 * time.Millisecond + // DB represents a collection of buckets persisted to a file on disk. // All data access is performed through transactions which can be obtained through the DB. // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. @@ -61,6 +65,11 @@ type DB struct { // THIS IS UNSAFE. PLEASE USE WITH CAUTION. NoSync bool + // When true, skips syncing freelist to disk. This improves the database + // write performance under normal operation, but requires a full database + // re-sync during recovery. + NoFreelistSync bool + // When true, skips the truncate call when growing the database. // Setting this to true is only safe on non-ext3/ext4 systems. // Skipping truncation avoids preallocation of hard drive space and @@ -96,8 +105,7 @@ type DB struct { path string file *os.File - lockfile *os.File // windows only - dataref []byte // mmap'ed readonly, write throws SEGV + dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int filesz int // current on disk file size @@ -107,9 +115,11 @@ type DB struct { opened bool rwtx *Tx txs []*Tx - freelist *freelist stats Stats + freelist *freelist + freelistLoad sync.Once + pagePool sync.Pool batchMu sync.Mutex @@ -148,14 +158,17 @@ func (db *DB) String() string { // If the file does not exist then it will be created automatically. // Passing in nil options will cause Bolt to open the database with the default options. func Open(path string, mode os.FileMode, options *Options) (*DB, error) { - var db = &DB{opened: true} - + db := &DB{ + opened: true, + } // Set default options if no options are provided. if options == nil { options = DefaultOptions } + db.NoSync = options.NoSync db.NoGrowSync = options.NoGrowSync db.MmapFlags = options.MmapFlags + db.NoFreelistSync = options.NoFreelistSync // Set default values for later DB operations. db.MaxBatchSize = DefaultMaxBatchSize @@ -183,7 +196,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // if !options.ReadOnly. // The database file is locked using the shared lock (more than one process may // hold a lock at the same time) otherwise (options.ReadOnly is set). - if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil { + if err := flock(db, !db.readOnly, options.Timeout); err != nil { _ = db.close() return nil, err } @@ -191,31 +204,41 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // Default values for test hooks db.ops.writeAt = db.file.WriteAt + if db.pageSize = options.PageSize; db.pageSize == 0 { + // Set the default page size to the OS page size. + db.pageSize = defaultPageSize + } + // Initialize the database if it doesn't exist. if info, err := db.file.Stat(); err != nil { + _ = db.close() return nil, err } else if info.Size() == 0 { // Initialize new files with meta pages. if err := db.init(); err != nil { + // clean up file descriptor on initialization fail + _ = db.close() return nil, err } } else { // Read the first meta page to determine the page size. var buf [0x1000]byte - if _, err := db.file.ReadAt(buf[:], 0); err == nil { - m := db.pageInBuffer(buf[:], 0).meta() - if err := m.validate(); err != nil { - // If we can't read the page size, we can assume it's the same - // as the OS -- since that's how the page size was chosen in the - // first place. - // - // If the first page is invalid and this OS uses a different - // page size than what the database was created with then we - // are out of luck and cannot access the database. - db.pageSize = os.Getpagesize() - } else { + // If we can't read the page size, but can read a page, assume + // it's the same as the OS or one given -- since that's how the + // page size was chosen in the first place. + // + // If the first page is invalid and this OS uses a different + // page size than what the database was created with then we + // are out of luck and cannot access the database. + // + // TODO: scan for next page + if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) { + if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil { db.pageSize = int(m.pageSize) } + } else { + _ = db.close() + return nil, ErrInvalid } } @@ -232,14 +255,50 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { return nil, err } - // Read in the freelist. - db.freelist = newFreelist() - db.freelist.read(db.page(db.meta().freelist)) + if db.readOnly { + return db, nil + } + + db.loadFreelist() + + // Flush freelist when transitioning from no sync to sync so + // NoFreelistSync unaware boltdb can open the db later. + if !db.NoFreelistSync && !db.hasSyncedFreelist() { + tx, err := db.Begin(true) + if tx != nil { + err = tx.Commit() + } + if err != nil { + _ = db.close() + return nil, err + } + } // Mark the database as opened and return. return db, nil } +// loadFreelist reads the freelist if it is synced, or reconstructs it +// by scanning the DB if it is not synced. It assumes there are no +// concurrent accesses being made to the freelist. +func (db *DB) loadFreelist() { + db.freelistLoad.Do(func() { + db.freelist = newFreelist() + if !db.hasSyncedFreelist() { + // Reconstruct free list by scanning the DB. + db.freelist.readIDs(db.freepages()) + } else { + // Read free list from freelist page. + db.freelist.read(db.page(db.meta().freelist)) + } + db.stats.FreePageN = len(db.freelist.ids) + }) +} + +func (db *DB) hasSyncedFreelist() bool { + return db.meta().freelist != pgidNoFreelist +} + // mmap opens the underlying memory-mapped file and initializes the meta references. // minsz is the minimum size that the new mmap can be. func (db *DB) mmap(minsz int) error { @@ -341,9 +400,6 @@ func (db *DB) mmapSize(size int) (int, error) { // init creates a new database file and initializes its meta pages. func (db *DB) init() error { - // Set the page size to the OS page size. - db.pageSize = os.Getpagesize() - // Create two meta pages on a buffer. buf := make([]byte, db.pageSize*4) for i := 0; i < 2; i++ { @@ -387,7 +443,8 @@ func (db *DB) init() error { } // Close releases all database resources. -// All transactions must be closed before closing the database. +// It will block waiting for any open transactions to finish +// before closing the database and returning. func (db *DB) Close() error { db.rwlock.Lock() defer db.rwlock.Unlock() @@ -395,8 +452,8 @@ func (db *DB) Close() error { db.metalock.Lock() defer db.metalock.Unlock() - db.mmaplock.RLock() - defer db.mmaplock.RUnlock() + db.mmaplock.Lock() + defer db.mmaplock.Unlock() return db.close() } @@ -526,21 +583,36 @@ func (db *DB) beginRWTx() (*Tx, error) { t := &Tx{writable: true} t.init(db) db.rwtx = t + db.freePages() + return t, nil +} - // Free any pages associated with closed read-only transactions. - var minid txid = 0xFFFFFFFFFFFFFFFF - for _, t := range db.txs { - if t.meta.txid < minid { - minid = t.meta.txid - } +// freePages releases any pages associated with closed read-only transactions. +func (db *DB) freePages() { + // Free all pending pages prior to earliest open transaction. + sort.Sort(txsById(db.txs)) + minid := txid(0xFFFFFFFFFFFFFFFF) + if len(db.txs) > 0 { + minid = db.txs[0].meta.txid } if minid > 0 { db.freelist.release(minid - 1) } - - return t, nil + // Release unused txid extents. + for _, t := range db.txs { + db.freelist.releaseRange(minid, t.meta.txid-1) + minid = t.meta.txid + 1 + } + db.freelist.releaseRange(minid, txid(0xFFFFFFFFFFFFFFFF)) + // Any page both allocated and freed in an extent is safe to release. } +type txsById []*Tx + +func (t txsById) Len() int { return len(t) } +func (t txsById) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t txsById) Less(i, j int) bool { return t[i].meta.txid < t[j].meta.txid } + // removeTx removes a transaction from the database. func (db *DB) removeTx(tx *Tx) { // Release the read lock on the mmap. @@ -633,11 +705,7 @@ func (db *DB) View(fn func(*Tx) error) error { return err } - if err := t.Rollback(); err != nil { - return err - } - - return nil + return t.Rollback() } // Batch calls fn as part of a batch. It behaves similar to Update, @@ -737,9 +805,7 @@ retry: // pass success, or bolt internal errors, to all callers for _, c := range b.calls { - if c.err != nil { - c.err <- err - } + c.err <- err } break retry } @@ -826,7 +892,7 @@ func (db *DB) meta() *meta { } // allocate returns a contiguous block of memory starting at a given page. -func (db *DB) allocate(count int) (*page, error) { +func (db *DB) allocate(txid txid, count int) (*page, error) { // Allocate a temporary buffer for the page. var buf []byte if count == 1 { @@ -838,7 +904,7 @@ func (db *DB) allocate(count int) (*page, error) { p.overflow = uint32(count - 1) // Use pages from the freelist if they are available. - if p.id = db.freelist.allocate(count); p.id != 0 { + if p.id = db.freelist.allocate(txid, count); p.id != 0 { return p, nil } @@ -893,6 +959,38 @@ func (db *DB) IsReadOnly() bool { return db.readOnly } +func (db *DB) freepages() []pgid { + tx, err := db.beginTx() + defer func() { + err = tx.Rollback() + if err != nil { + panic("freepages: failed to rollback tx") + } + }() + if err != nil { + panic("freepages: failed to open read only tx") + } + + reachable := make(map[pgid]*page) + nofreed := make(map[pgid]bool) + ech := make(chan error) + go func() { + for e := range ech { + panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e)) + } + }() + tx.checkBucket(&tx.root, reachable, nofreed, ech) + close(ech) + + var fids []pgid + for i := pgid(2); i < db.meta().pgid; i++ { + if _, ok := reachable[i]; !ok { + fids = append(fids, i) + } + } + return fids +} + // Options represents the options that can be set when opening a database. type Options struct { // Timeout is the amount of time to wait to obtain a file lock. @@ -903,6 +1001,10 @@ type Options struct { // Sets the DB.NoGrowSync flag before memory mapping the file. NoGrowSync bool + // Do not sync freelist to disk. This improves the database write performance + // under normal operation, but requires a full database re-sync during recovery. + NoFreelistSync bool + // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to // grab a shared lock (UNIX). ReadOnly bool @@ -919,6 +1021,14 @@ type Options struct { // If initialMmapSize is smaller than the previous database size, // it takes no effect. InitialMmapSize int + + // PageSize overrides the default OS page size. + PageSize int + + // NoSync sets the initial value of DB.NoSync. Normally this can just be + // set directly on the DB itself when returned from Open(), but this option + // is useful in APIs which expose Options but not the underlying DB. + NoSync bool } // DefaultOptions represent the options used if nil options are passed into Open(). @@ -960,10 +1070,6 @@ func (s *Stats) Sub(other *Stats) Stats { return diff } -func (s *Stats) add(other *Stats) { - s.TxStats.add(&other.TxStats) -} - type Info struct { Data uintptr PageSize int @@ -1002,7 +1108,8 @@ func (m *meta) copy(dest *meta) { func (m *meta) write(p *page) { if m.root.root >= m.pgid { panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid)) - } else if m.freelist >= m.pgid { + } else if m.freelist >= m.pgid && m.freelist != pgidNoFreelist { + // TODO: reject pgidNoFreeList if !NoFreelistSync panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid)) } @@ -1029,11 +1136,3 @@ func _assert(condition bool, msg string, v ...interface{}) { panic(fmt.Sprintf("assertion failed: "+msg, v...)) } } - -func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } -func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } - -func printstack() { - stack := strings.Join(strings.Split(string(debug.Stack()), "\n")[2:], "\n") - fmt.Fprintln(os.Stderr, stack) -} diff --git a/vendor/github.com/boltdb/bolt/doc.go b/vendor/go.etcd.io/bbolt/doc.go similarity index 94% rename from vendor/github.com/boltdb/bolt/doc.go rename to vendor/go.etcd.io/bbolt/doc.go index cc937845d..95f25f01c 100644 --- a/vendor/github.com/boltdb/bolt/doc.go +++ b/vendor/go.etcd.io/bbolt/doc.go @@ -1,5 +1,5 @@ /* -Package bolt implements a low-level key/value store in pure Go. It supports +package bbolt implements a low-level key/value store in pure Go. It supports fully serializable transactions, ACID semantics, and lock-free MVCC with multiple readers and a single writer. Bolt can be used for projects that want a simple data store without the need to add large dependencies such as @@ -41,4 +41,4 @@ point to different data or can point to invalid memory which will cause a panic. */ -package bolt +package bbolt diff --git a/vendor/github.com/boltdb/bolt/errors.go b/vendor/go.etcd.io/bbolt/errors.go similarity index 99% rename from vendor/github.com/boltdb/bolt/errors.go rename to vendor/go.etcd.io/bbolt/errors.go index a3620a3eb..48758ca57 100644 --- a/vendor/github.com/boltdb/bolt/errors.go +++ b/vendor/go.etcd.io/bbolt/errors.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "errors" diff --git a/vendor/github.com/boltdb/bolt/freelist.go b/vendor/go.etcd.io/bbolt/freelist.go similarity index 65% rename from vendor/github.com/boltdb/bolt/freelist.go rename to vendor/go.etcd.io/bbolt/freelist.go index aba48f58c..e4bcb2dcf 100644 --- a/vendor/github.com/boltdb/bolt/freelist.go +++ b/vendor/go.etcd.io/bbolt/freelist.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -6,18 +6,28 @@ import ( "unsafe" ) +// txPending holds a list of pgids and corresponding allocation txns +// that are pending to be freed. +type txPending struct { + ids []pgid + alloctx []txid // txids allocating the ids + lastReleaseBegin txid // beginning txid of last matching releaseRange +} + // freelist represents a list of all pages that are available for allocation. // It also tracks pages that have been freed but are still in use by open transactions. type freelist struct { - ids []pgid // all free and available free page ids. - pending map[txid][]pgid // mapping of soon-to-be free page ids by tx. - cache map[pgid]bool // fast lookup of all free and pending page ids. + ids []pgid // all free and available free page ids. + allocs map[pgid]txid // mapping of txid that allocated a pgid. + pending map[txid]*txPending // mapping of soon-to-be free page ids by tx. + cache map[pgid]bool // fast lookup of all free and pending page ids. } // newFreelist returns an empty, initialized freelist. func newFreelist() *freelist { return &freelist{ - pending: make(map[txid][]pgid), + allocs: make(map[pgid]txid), + pending: make(map[txid]*txPending), cache: make(map[pgid]bool), } } @@ -45,8 +55,8 @@ func (f *freelist) free_count() int { // pending_count returns count of pending pages func (f *freelist) pending_count() int { var count int - for _, list := range f.pending { - count += len(list) + for _, txp := range f.pending { + count += len(txp.ids) } return count } @@ -55,8 +65,8 @@ func (f *freelist) pending_count() int { // f.count returns the minimum length required for dst. func (f *freelist) copyall(dst []pgid) { m := make(pgids, 0, f.pending_count()) - for _, list := range f.pending { - m = append(m, list...) + for _, txp := range f.pending { + m = append(m, txp.ids...) } sort.Sort(m) mergepgids(dst, f.ids, m) @@ -64,7 +74,7 @@ func (f *freelist) copyall(dst []pgid) { // allocate returns the starting page id of a contiguous list of pages of a given size. // If a contiguous block cannot be found then 0 is returned. -func (f *freelist) allocate(n int) pgid { +func (f *freelist) allocate(txid txid, n int) pgid { if len(f.ids) == 0 { return 0 } @@ -97,7 +107,7 @@ func (f *freelist) allocate(n int) pgid { for i := pgid(0); i < pgid(n); i++ { delete(f.cache, initial+i) } - + f.allocs[initial] = txid return initial } @@ -114,28 +124,73 @@ func (f *freelist) free(txid txid, p *page) { } // Free page and all its overflow pages. - var ids = f.pending[txid] + txp := f.pending[txid] + if txp == nil { + txp = &txPending{} + f.pending[txid] = txp + } + allocTxid, ok := f.allocs[p.id] + if ok { + delete(f.allocs, p.id) + } else if (p.flags & freelistPageFlag) != 0 { + // Freelist is always allocated by prior tx. + allocTxid = txid - 1 + } + for id := p.id; id <= p.id+pgid(p.overflow); id++ { // Verify that page is not already free. if f.cache[id] { panic(fmt.Sprintf("page %d already freed", id)) } - // Add to the freelist and cache. - ids = append(ids, id) + txp.ids = append(txp.ids, id) + txp.alloctx = append(txp.alloctx, allocTxid) f.cache[id] = true } - f.pending[txid] = ids } // release moves all page ids for a transaction id (or older) to the freelist. func (f *freelist) release(txid txid) { m := make(pgids, 0) - for tid, ids := range f.pending { + for tid, txp := range f.pending { if tid <= txid { // Move transaction's pending pages to the available freelist. // Don't remove from the cache since the page is still free. - m = append(m, ids...) + m = append(m, txp.ids...) + delete(f.pending, tid) + } + } + sort.Sort(m) + f.ids = pgids(f.ids).merge(m) +} + +// releaseRange moves pending pages allocated within an extent [begin,end] to the free list. +func (f *freelist) releaseRange(begin, end txid) { + if begin > end { + return + } + var m pgids + for tid, txp := range f.pending { + if tid < begin || tid > end { + continue + } + // Don't recompute freed pages if ranges haven't updated. + if txp.lastReleaseBegin == begin { + continue + } + for i := 0; i < len(txp.ids); i++ { + if atx := txp.alloctx[i]; atx < begin || atx > end { + continue + } + m = append(m, txp.ids[i]) + txp.ids[i] = txp.ids[len(txp.ids)-1] + txp.ids = txp.ids[:len(txp.ids)-1] + txp.alloctx[i] = txp.alloctx[len(txp.alloctx)-1] + txp.alloctx = txp.alloctx[:len(txp.alloctx)-1] + i-- + } + txp.lastReleaseBegin = begin + if len(txp.ids) == 0 { delete(f.pending, tid) } } @@ -146,12 +201,29 @@ func (f *freelist) release(txid txid) { // rollback removes the pages from a given pending tx. func (f *freelist) rollback(txid txid) { // Remove page ids from cache. - for _, id := range f.pending[txid] { - delete(f.cache, id) + txp := f.pending[txid] + if txp == nil { + return } - - // Remove pages from pending list. + var m pgids + for i, pgid := range txp.ids { + delete(f.cache, pgid) + tx := txp.alloctx[i] + if tx == 0 { + continue + } + if tx != txid { + // Pending free aborted; restore page back to alloc list. + f.allocs[pgid] = tx + } else { + // Freed page was allocated by this txn; OK to throw away. + m = append(m, pgid) + } + } + // Remove pages from pending list and mark as free if allocated by txid. delete(f.pending, txid) + sort.Sort(m) + f.ids = pgids(f.ids).merge(m) } // freed returns whether a given page is in the free list. @@ -161,6 +233,9 @@ func (f *freelist) freed(pgid pgid) bool { // read initializes the freelist from a freelist page. func (f *freelist) read(p *page) { + if (p.flags & freelistPageFlag) == 0 { + panic(fmt.Sprintf("invalid freelist page: %d, page type is %s", p.id, p.typ())) + } // If the page.count is at the max uint16 value (64k) then it's considered // an overflow and the size of the freelist is stored as the first element. idx, count := 0, int(p.count) @@ -173,7 +248,7 @@ func (f *freelist) read(p *page) { if count == 0 { f.ids = nil } else { - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count] f.ids = make([]pgid, len(ids)) copy(f.ids, ids) @@ -185,6 +260,12 @@ func (f *freelist) read(p *page) { f.reindex() } +// read initializes the freelist from a given list of ids. +func (f *freelist) readIDs(ids []pgid) { + f.ids = ids + f.reindex() +} + // write writes the page ids onto a freelist page. All free and pending ids are // saved to disk since in the event of a program crash, all pending ids will // become free. @@ -217,8 +298,8 @@ func (f *freelist) reload(p *page) { // Build a cache of only pending pages. pcache := make(map[pgid]bool) - for _, pendingIDs := range f.pending { - for _, pendingID := range pendingIDs { + for _, txp := range f.pending { + for _, pendingID := range txp.ids { pcache[pendingID] = true } } @@ -244,8 +325,8 @@ func (f *freelist) reindex() { for _, id := range f.ids { f.cache[id] = true } - for _, pendingIDs := range f.pending { - for _, pendingID := range pendingIDs { + for _, txp := range f.pending { + for _, pendingID := range txp.ids { f.cache[pendingID] = true } } diff --git a/vendor/github.com/boltdb/bolt/node.go b/vendor/go.etcd.io/bbolt/node.go similarity index 99% rename from vendor/github.com/boltdb/bolt/node.go rename to vendor/go.etcd.io/bbolt/node.go index 159318b22..6c3fa553e 100644 --- a/vendor/github.com/boltdb/bolt/node.go +++ b/vendor/go.etcd.io/bbolt/node.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -365,7 +365,7 @@ func (n *node) spill() error { } // Allocate contiguous space for the node. - p, err := tx.allocate((node.size() / tx.db.pageSize) + 1) + p, err := tx.allocate((node.size() + tx.db.pageSize - 1) / tx.db.pageSize) if err != nil { return err } diff --git a/vendor/github.com/boltdb/bolt/page.go b/vendor/go.etcd.io/bbolt/page.go similarity index 99% rename from vendor/github.com/boltdb/bolt/page.go rename to vendor/go.etcd.io/bbolt/page.go index cde403ae8..bca9615f0 100644 --- a/vendor/github.com/boltdb/bolt/page.go +++ b/vendor/go.etcd.io/bbolt/page.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" diff --git a/vendor/github.com/boltdb/bolt/tx.go b/vendor/go.etcd.io/bbolt/tx.go similarity index 94% rename from vendor/github.com/boltdb/bolt/tx.go rename to vendor/go.etcd.io/bbolt/tx.go index 6700308a2..f50864142 100644 --- a/vendor/github.com/boltdb/bolt/tx.go +++ b/vendor/go.etcd.io/bbolt/tx.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -126,10 +126,7 @@ func (tx *Tx) DeleteBucket(name []byte) error { // the error is returned to the caller. func (tx *Tx) ForEach(fn func(name []byte, b *Bucket) error) error { return tx.root.ForEach(func(k, v []byte) error { - if err := fn(k, tx.root.Bucket(k)); err != nil { - return err - } - return nil + return fn(k, tx.root.Bucket(k)) }) } @@ -169,28 +166,18 @@ func (tx *Tx) Commit() error { // Free the old root bucket. tx.meta.root.root = tx.root.root - opgid := tx.meta.pgid - - // Free the freelist and allocate new pages for it. This will overestimate - // the size of the freelist but not underestimate the size (which would be bad). - tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) - p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) - if err != nil { - tx.rollback() - return err + // Free the old freelist because commit writes out a fresh freelist. + if tx.meta.freelist != pgidNoFreelist { + tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) } - if err := tx.db.freelist.write(p); err != nil { - tx.rollback() - return err - } - tx.meta.freelist = p.id - // If the high water mark has moved up then attempt to grow the database. - if tx.meta.pgid > opgid { - if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { - tx.rollback() + if !tx.db.NoFreelistSync { + err := tx.commitFreelist() + if err != nil { return err } + } else { + tx.meta.freelist = pgidNoFreelist } // Write dirty pages to disk. @@ -235,6 +222,31 @@ func (tx *Tx) Commit() error { return nil } +func (tx *Tx) commitFreelist() error { + // Allocate new pages for the new free list. This will overestimate + // the size of the freelist but not underestimate the size (which would be bad). + opgid := tx.meta.pgid + p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) + if err != nil { + tx.rollback() + return err + } + if err := tx.db.freelist.write(p); err != nil { + tx.rollback() + return err + } + tx.meta.freelist = p.id + // If the high water mark has moved up then attempt to grow the database. + if tx.meta.pgid > opgid { + if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { + tx.rollback() + return err + } + } + + return nil +} + // Rollback closes the transaction and ignores all previous updates. Read-only // transactions must be rolled back and not committed. func (tx *Tx) Rollback() error { @@ -291,7 +303,9 @@ func (tx *Tx) close() { } // Copy writes the entire database to a writer. -// This function exists for backwards compatibility. Use WriteTo() instead. +// This function exists for backwards compatibility. +// +// Deprecated; Use WriteTo() instead. func (tx *Tx) Copy(w io.Writer) error { _, err := tx.WriteTo(w) return err @@ -305,7 +319,11 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { if err != nil { return 0, err } - defer func() { _ = f.Close() }() + defer func() { + if cerr := f.Close(); err == nil { + err = cerr + } + }() // Generate a meta page. We use the same page data for both meta pages. buf := make([]byte, tx.db.pageSize) @@ -333,7 +351,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { } // Move past the meta pages in the file. - if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil { + if _, err := f.Seek(int64(tx.db.pageSize*2), io.SeekStart); err != nil { return n, fmt.Errorf("seek: %s", err) } @@ -344,7 +362,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { return n, err } - return n, f.Close() + return n, nil } // CopyFile copies the entire database to file at the given path. @@ -379,6 +397,9 @@ func (tx *Tx) Check() <-chan error { } func (tx *Tx) check(ch chan error) { + // Force loading free list if opened in ReadOnly mode. + tx.db.loadFreelist() + // Check if any pages are double freed. freed := make(map[pgid]bool) all := make([]pgid, tx.db.freelist.count()) @@ -394,8 +415,10 @@ func (tx *Tx) check(ch chan error) { reachable := make(map[pgid]*page) reachable[0] = tx.page(0) // meta0 reachable[1] = tx.page(1) // meta1 - for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { - reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) + if tx.meta.freelist != pgidNoFreelist { + for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { + reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) + } } // Recursively check buckets. @@ -453,7 +476,7 @@ func (tx *Tx) checkBucket(b *Bucket, reachable map[pgid]*page, freed map[pgid]bo // allocate returns a contiguous block of memory starting at a given page. func (tx *Tx) allocate(count int) (*page, error) { - p, err := tx.db.allocate(count) + p, err := tx.db.allocate(tx.meta.txid, count) if err != nil { return nil, err } @@ -462,7 +485,7 @@ func (tx *Tx) allocate(count int) (*page, error) { tx.pages[p.id] = p // Update statistics. - tx.stats.PageCount++ + tx.stats.PageCount += count tx.stats.PageAlloc += count * tx.db.pageSize return p, nil