Merge pull request #1276 from dmcgowan/snapshot-labels

Snapshot labels
This commit is contained in:
Michael Crosby 2017-08-08 21:59:29 -04:00 committed by GitHub
commit 29a4dd7f46
21 changed files with 2416 additions and 923 deletions

File diff suppressed because it is too large Load Diff

View File

@ -4,6 +4,8 @@ package containerd.services.snapshots.v1;
import "gogoproto/gogo.proto"; import "gogoproto/gogo.proto";
import "google/protobuf/empty.proto"; import "google/protobuf/empty.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";
import "github.com/containerd/containerd/api/types/mount.proto"; import "github.com/containerd/containerd/api/types/mount.proto";
option go_package = "github.com/containerd/containerd/api/services/snapshot/v1;snapshot"; option go_package = "github.com/containerd/containerd/api/services/snapshot/v1;snapshot";
@ -16,6 +18,7 @@ service Snapshots {
rpc Commit(CommitSnapshotRequest) returns (google.protobuf.Empty); rpc Commit(CommitSnapshotRequest) returns (google.protobuf.Empty);
rpc Remove(RemoveSnapshotRequest) returns (google.protobuf.Empty); rpc Remove(RemoveSnapshotRequest) returns (google.protobuf.Empty);
rpc Stat(StatSnapshotRequest) returns (StatSnapshotResponse); rpc Stat(StatSnapshotRequest) returns (StatSnapshotResponse);
rpc Update(UpdateSnapshotRequest) returns (UpdateSnapshotResponse);
rpc List(ListSnapshotsRequest) returns (stream ListSnapshotsResponse); rpc List(ListSnapshotsRequest) returns (stream ListSnapshotsResponse);
rpc Usage(UsageRequest) returns (UsageResponse); rpc Usage(UsageRequest) returns (UsageResponse);
} }
@ -24,6 +27,9 @@ message PrepareSnapshotRequest {
string snapshotter = 1; string snapshotter = 1;
string key = 2; string key = 2;
string parent = 3; string parent = 3;
// Labels are arbitrary data on snapshots.
map<string, string> labels = 4;
} }
message PrepareSnapshotResponse { message PrepareSnapshotResponse {
@ -34,6 +40,9 @@ message ViewSnapshotRequest {
string snapshotter = 1; string snapshotter = 1;
string key = 2; string key = 2;
string parent = 3; string parent = 3;
// Labels are arbitrary data on snapshots.
map<string, string> labels = 4;
} }
message ViewSnapshotResponse { message ViewSnapshotResponse {
@ -58,6 +67,9 @@ message CommitSnapshotRequest {
string snapshotter = 1; string snapshotter = 1;
string name = 2; string name = 2;
string key = 3; string key = 3;
// Labels are arbitrary data on snapshots.
map<string, string> labels = 4;
} }
message StatSnapshotRequest { message StatSnapshotRequest {
@ -79,12 +91,38 @@ message Info {
string name = 1; string name = 1;
string parent = 2; string parent = 2;
Kind kind = 3; Kind kind = 3;
// CreatedAt provides the time at which the snapshot was created.
google.protobuf.Timestamp created_at = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
// UpdatedAt provides the time the info was last updated.
google.protobuf.Timestamp updated_at = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
// Labels are arbitrary data on snapshots.
map<string, string> labels = 6;
} }
message StatSnapshotResponse { message StatSnapshotResponse {
Info info = 1 [(gogoproto.nullable) = false]; Info info = 1 [(gogoproto.nullable) = false];
} }
message UpdateSnapshotRequest {
string snapshotter = 1;
Info info = 2 [(gogoproto.nullable) = false];
// UpdateMask specifies which fields to perform the update on. If empty,
// the operation applies to all fields.
//
// In info, Name, Parent, Kind, Created are immutable,
// other field may be updated using this mask.
// If no mask is provided, all mutable field are updated.
google.protobuf.FieldMask update_mask = 3;
}
message UpdateSnapshotResponse {
Info info = 1 [(gogoproto.nullable) = false];
}
message ListSnapshotsRequest{ message ListSnapshotsRequest{
string snapshotter = 1; string snapshotter = 1;
} }

View File

@ -1,4 +1,4 @@
package metadata package boltutil
import ( import (
"time" "time"
@ -7,19 +7,36 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
func readLabels(m map[string]string, bkt *bolt.Bucket) error { var (
return bkt.ForEach(func(k, v []byte) error { bucketKeyLabels = []byte("labels")
m[string(k)] = string(v) bucketKeyCreatedAt = []byte("createdat")
bucketKeyUpdatedAt = []byte("updatedat")
)
// ReadLabels reads the labels key from the bucket
// Uses the key "labels"
func ReadLabels(bkt *bolt.Bucket) (map[string]string, error) {
lbkt := bkt.Bucket(bucketKeyLabels)
if lbkt == nil {
return nil, nil
}
labels := map[string]string{}
if err := lbkt.ForEach(func(k, v []byte) error {
labels[string(k)] = string(v)
return nil return nil
}) }); err != nil {
return nil, err
}
return labels, nil
} }
// writeLabels will write a new labels bucket to the provided bucket at key // WriteLabels will write a new labels bucket to the provided bucket at key
// bucketKeyLabels, replacing the contents of the bucket with the provided map. // bucketKeyLabels, replacing the contents of the bucket with the provided map.
// //
// The provide map labels will be modified to have the final contents of the // The provide map labels will be modified to have the final contents of the
// bucket. Typically, this removes zero-value entries. // bucket. Typically, this removes zero-value entries.
func writeLabels(bkt *bolt.Bucket, labels map[string]string) error { // Uses the key "labels"
func WriteLabels(bkt *bolt.Bucket, labels map[string]string) error {
// Remove existing labels to keep from merging // Remove existing labels to keep from merging
if lbkt := bkt.Bucket(bucketKeyLabels); lbkt != nil { if lbkt := bkt.Bucket(bucketKeyLabels); lbkt != nil {
if err := bkt.DeleteBucket(bucketKeyLabels); err != nil { if err := bkt.DeleteBucket(bucketKeyLabels); err != nil {
@ -50,7 +67,9 @@ func writeLabels(bkt *bolt.Bucket, labels map[string]string) error {
return nil return nil
} }
func readTimestamps(created, updated *time.Time, bkt *bolt.Bucket) error { // ReadTimestamps reads created and updated timestamps from a bucket.
// Uses keys "createdat" and "updatedat"
func ReadTimestamps(bkt *bolt.Bucket, created, updated *time.Time) error {
for _, f := range []struct { for _, f := range []struct {
b []byte b []byte
t *time.Time t *time.Time
@ -68,7 +87,9 @@ func readTimestamps(created, updated *time.Time, bkt *bolt.Bucket) error {
return nil return nil
} }
func writeTimestamps(bkt *bolt.Bucket, created, updated time.Time) error { // WriteTimestamps writes created and updated timestamps to a bucket.
// Uses keys "createdat" and "updatedat"
func WriteTimestamps(bkt *bolt.Bucket, created, updated time.Time) error {
createdAt, err := created.MarshalBinary() createdAt, err := created.MarshalBinary()
if err != nil { if err != nil {
return err return err

View File

@ -41,16 +41,14 @@ var (
bucketKeyDigest = []byte("digest") bucketKeyDigest = []byte("digest")
bucketKeyMediaType = []byte("mediatype") bucketKeyMediaType = []byte("mediatype")
bucketKeySize = []byte("size") bucketKeySize = []byte("size")
bucketKeyLabels = []byte("labels")
bucketKeyImage = []byte("image") bucketKeyImage = []byte("image")
bucketKeyRuntime = []byte("runtime") bucketKeyRuntime = []byte("runtime")
bucketKeyName = []byte("name") bucketKeyName = []byte("name")
bucketKeyParent = []byte("parent")
bucketKeyOptions = []byte("options") bucketKeyOptions = []byte("options")
bucketKeySpec = []byte("spec") bucketKeySpec = []byte("spec")
bucketKeyRootFS = []byte("rootfs") bucketKeyRootFS = []byte("rootfs")
bucketKeyTarget = []byte("target") bucketKeyTarget = []byte("target")
bucketKeyCreatedAt = []byte("createdat")
bucketKeyUpdatedAt = []byte("updatedat")
) )
func getBucket(tx *bolt.Tx, keys ...[]byte) *bolt.Bucket { func getBucket(tx *bolt.Tx, keys ...[]byte) *bolt.Bucket {

View File

@ -10,6 +10,7 @@ import (
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/filters" "github.com/containerd/containerd/filters"
"github.com/containerd/containerd/identifiers" "github.com/containerd/containerd/identifiers"
"github.com/containerd/containerd/metadata/boltutil"
"github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/namespaces"
"github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/proto"
"github.com/gogo/protobuf/types" "github.com/gogo/protobuf/types"
@ -206,6 +207,16 @@ func (s *containerStore) Delete(ctx context.Context, id string) error {
} }
func readContainer(container *containers.Container, bkt *bolt.Bucket) error { func readContainer(container *containers.Container, bkt *bolt.Bucket) error {
labels, err := boltutil.ReadLabels(bkt)
if err != nil {
return err
}
container.Labels = labels
if err := boltutil.ReadTimestamps(bkt, &container.CreatedAt, &container.UpdatedAt); err != nil {
return err
}
return bkt.ForEach(func(k, v []byte) error { return bkt.ForEach(func(k, v []byte) error {
switch string(k) { switch string(k) {
case string(bucketKeyImage): case string(bucketKeyImage):
@ -239,24 +250,7 @@ func readContainer(container *containers.Container, bkt *bolt.Bucket) error {
container.Spec = &any container.Spec = &any
case string(bucketKeyRootFS): case string(bucketKeyRootFS):
container.RootFS = string(v) container.RootFS = string(v)
case string(bucketKeyCreatedAt):
if err := container.CreatedAt.UnmarshalBinary(v); err != nil {
return err
}
case string(bucketKeyUpdatedAt):
if err := container.UpdatedAt.UnmarshalBinary(v); err != nil {
return err
}
case string(bucketKeyLabels):
lbkt := bkt.Bucket(bucketKeyLabels)
if lbkt == nil {
return nil
}
container.Labels = map[string]string{}
if err := readLabels(container.Labels, lbkt); err != nil {
return err
}
} }
return nil return nil
@ -264,7 +258,7 @@ func readContainer(container *containers.Container, bkt *bolt.Bucket) error {
} }
func writeContainer(bkt *bolt.Bucket, container *containers.Container) error { func writeContainer(bkt *bolt.Bucket, container *containers.Container) error {
if err := writeTimestamps(bkt, container.CreatedAt, container.UpdatedAt); err != nil { if err := boltutil.WriteTimestamps(bkt, container.CreatedAt, container.UpdatedAt); err != nil {
return err return err
} }
@ -314,5 +308,5 @@ func writeContainer(bkt *bolt.Bucket, container *containers.Container) error {
} }
} }
return writeLabels(bkt, container.Labels) return boltutil.WriteLabels(bkt, container.Labels)
} }

View File

@ -11,6 +11,7 @@ import (
"github.com/containerd/containerd/content" "github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/filters" "github.com/containerd/containerd/filters"
"github.com/containerd/containerd/metadata/boltutil"
"github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/namespaces"
digest "github.com/opencontainers/go-digest" digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -390,24 +391,18 @@ func (nw *namespacedWriter) commit(tx *bolt.Tx, size int64, expected digest.Dige
return err return err
} }
commitTime := time.Now().UTC()
sizeEncoded, err := encodeSize(size) sizeEncoded, err := encodeSize(size)
if err != nil { if err != nil {
return err return err
} }
timeEncoded, err := time.Now().UTC().MarshalBinary() if err := boltutil.WriteTimestamps(bkt, commitTime, commitTime); err != nil {
if err != nil {
return err return err
} }
if err := bkt.Put(bucketKeySize, sizeEncoded); err != nil {
for _, v := range [][2][]byte{ return err
{bucketKeyCreatedAt, timeEncoded},
{bucketKeyUpdatedAt, timeEncoded},
{bucketKeySize, sizeEncoded},
} {
if err := bkt.Put(v[0], v[1]); err != nil {
return err
}
} }
return nil return nil
@ -451,17 +446,15 @@ func (cs *contentStore) checkAccess(ctx context.Context, dgst digest.Digest) err
} }
func readInfo(info *content.Info, bkt *bolt.Bucket) error { func readInfo(info *content.Info, bkt *bolt.Bucket) error {
if err := readTimestamps(&info.CreatedAt, &info.UpdatedAt, bkt); err != nil { if err := boltutil.ReadTimestamps(bkt, &info.CreatedAt, &info.UpdatedAt); err != nil {
return err return err
} }
lbkt := bkt.Bucket(bucketKeyLabels) labels, err := boltutil.ReadLabels(bkt)
if lbkt != nil { if err != nil {
info.Labels = map[string]string{} return err
if err := readLabels(info.Labels, lbkt); err != nil {
return err
}
} }
info.Labels = labels
if v := bkt.Get(bucketKeySize); len(v) > 0 { if v := bkt.Get(bucketKeySize); len(v) > 0 {
info.Size, _ = binary.Varint(v) info.Size, _ = binary.Varint(v)
@ -471,11 +464,11 @@ func readInfo(info *content.Info, bkt *bolt.Bucket) error {
} }
func writeInfo(info *content.Info, bkt *bolt.Bucket) error { func writeInfo(info *content.Info, bkt *bolt.Bucket) error {
if err := writeTimestamps(bkt, info.CreatedAt, info.UpdatedAt); err != nil { if err := boltutil.WriteTimestamps(bkt, info.CreatedAt, info.UpdatedAt); err != nil {
return err return err
} }
if err := writeLabels(bkt, info.Labels); err != nil { if err := boltutil.WriteLabels(bkt, info.Labels); err != nil {
return errors.Wrapf(err, "writing labels for info %v", info.Digest) return errors.Wrapf(err, "writing labels for info %v", info.Digest)
} }

View File

@ -11,6 +11,7 @@ import (
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/filters" "github.com/containerd/containerd/filters"
"github.com/containerd/containerd/images" "github.com/containerd/containerd/images"
"github.com/containerd/containerd/metadata/boltutil"
"github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/namespaces"
digest "github.com/opencontainers/go-digest" digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -191,17 +192,15 @@ func (s *imageStore) Delete(ctx context.Context, name string) error {
} }
func readImage(image *images.Image, bkt *bolt.Bucket) error { func readImage(image *images.Image, bkt *bolt.Bucket) error {
if err := readTimestamps(&image.CreatedAt, &image.UpdatedAt, bkt); err != nil { if err := boltutil.ReadTimestamps(bkt, &image.CreatedAt, &image.UpdatedAt); err != nil {
return err return err
} }
lbkt := bkt.Bucket(bucketKeyLabels) labels, err := boltutil.ReadLabels(bkt)
if lbkt != nil { if err != nil {
image.Labels = map[string]string{} return err
if err := readLabels(image.Labels, lbkt); err != nil {
return err
}
} }
image.Labels = labels
tbkt := bkt.Bucket(bucketKeyTarget) tbkt := bkt.Bucket(bucketKeyTarget)
if tbkt == nil { if tbkt == nil {
@ -228,11 +227,11 @@ func readImage(image *images.Image, bkt *bolt.Bucket) error {
} }
func writeImage(bkt *bolt.Bucket, image *images.Image) error { func writeImage(bkt *bolt.Bucket, image *images.Image) error {
if err := writeTimestamps(bkt, image.CreatedAt, image.UpdatedAt); err != nil { if err := boltutil.WriteTimestamps(bkt, image.CreatedAt, image.UpdatedAt); err != nil {
return err return err
} }
if err := writeLabels(bkt, image.Labels); err != nil { if err := boltutil.WriteLabels(bkt, image.Labels); err != nil {
return errors.Wrapf(err, "writing labels for image %v", image.Name) return errors.Wrapf(err, "writing labels for image %v", image.Name)
} }

View File

@ -4,9 +4,11 @@ import (
"context" "context"
"fmt" "fmt"
"strings" "strings"
"time"
"github.com/boltdb/bolt" "github.com/boltdb/bolt"
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/metadata/boltutil"
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
@ -46,7 +48,11 @@ func getKey(tx *bolt.Tx, ns, name, key string) string {
if bkt == nil { if bkt == nil {
return "" return ""
} }
v := bkt.Get([]byte(key)) bkt = bkt.Bucket([]byte(key))
if bkt == nil {
return ""
}
v := bkt.Get(bucketKeyName)
if len(v) == 0 { if len(v) == 0 {
return "" return ""
} }
@ -74,20 +80,144 @@ func (s *snapshotter) resolveKey(ctx context.Context, key string) (string, error
} }
func (s *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) { func (s *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) {
bkey, err := s.resolveKey(ctx, key) ns, err := namespaces.NamespaceRequired(ctx)
if err != nil { if err != nil {
return snapshot.Info{}, err return snapshot.Info{}, err
} }
var (
bkey string
local = snapshot.Info{
Name: key,
}
)
if err := view(ctx, s.db, func(tx *bolt.Tx) error {
bkt := getSnapshotterBucket(tx, ns, s.name)
if bkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key)
}
sbkt := bkt.Bucket([]byte(key))
if sbkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key)
}
local.Labels, err = boltutil.ReadLabels(sbkt)
if err != nil {
return errors.Wrap(err, "failed to read labels")
}
if err := boltutil.ReadTimestamps(sbkt, &local.Created, &local.Updated); err != nil {
return errors.Wrap(err, "failed to read timestamps")
}
bkey = string(sbkt.Get(bucketKeyName))
local.Parent = string(sbkt.Get(bucketKeyParent))
return nil
}); err != nil {
return snapshot.Info{}, err
}
info, err := s.Snapshotter.Stat(ctx, bkey) info, err := s.Snapshotter.Stat(ctx, bkey)
if err != nil { if err != nil {
return snapshot.Info{}, err return snapshot.Info{}, err
} }
info.Name = trimKey(info.Name)
if info.Parent != "" { return overlayInfo(info, local), nil
info.Parent = trimKey(info.Parent) }
func (s *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) {
ns, err := namespaces.NamespaceRequired(ctx)
if err != nil {
return snapshot.Info{}, err
} }
return info, nil if info.Name == "" {
return snapshot.Info{}, errors.Wrap(errdefs.ErrInvalidArgument, "")
}
var (
bkey string
local = snapshot.Info{
Name: info.Name,
}
)
if err := update(ctx, s.db, func(tx *bolt.Tx) error {
bkt := getSnapshotterBucket(tx, ns, s.name)
if bkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", info.Name)
}
sbkt := bkt.Bucket([]byte(info.Name))
if sbkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", info.Name)
}
local.Labels, err = boltutil.ReadLabels(sbkt)
if err != nil {
return errors.Wrap(err, "failed to read labels")
}
if err := boltutil.ReadTimestamps(sbkt, &local.Created, &local.Updated); err != nil {
return errors.Wrap(err, "failed to read timestamps")
}
// Handle field updates
if len(fieldpaths) > 0 {
for _, path := range fieldpaths {
if strings.HasPrefix(path, "labels.") {
if local.Labels == nil {
local.Labels = map[string]string{}
}
key := strings.TrimPrefix(path, "labels.")
local.Labels[key] = info.Labels[key]
continue
}
switch path {
case "labels":
local.Labels = info.Labels
default:
return errors.Wrapf(errdefs.ErrInvalidArgument, "cannot update %q field on snapshot %q", path, info.Name)
}
}
} else {
local.Labels = info.Labels
}
local.Updated = time.Now().UTC()
if err := boltutil.WriteTimestamps(sbkt, local.Created, local.Updated); err != nil {
return errors.Wrap(err, "failed to read timestamps")
}
if err := boltutil.WriteLabels(sbkt, local.Labels); err != nil {
return errors.Wrap(err, "failed to read labels")
}
bkey = string(sbkt.Get(bucketKeyName))
local.Parent = string(sbkt.Get(bucketKeyParent))
return nil
}); err != nil {
return snapshot.Info{}, err
}
info, err = s.Snapshotter.Stat(ctx, bkey)
if err != nil {
return snapshot.Info{}, err
}
return overlayInfo(info, local), nil
}
func overlayInfo(info, overlay snapshot.Info) snapshot.Info {
// Merge info
info.Name = overlay.Name
info.Created = overlay.Created
info.Updated = overlay.Updated
info.Parent = overlay.Parent
if info.Labels == nil {
info.Labels = overlay.Labels
} else {
for k, v := range overlay.Labels {
overlay.Labels[k] = v
}
}
return info
} }
func (s *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { func (s *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) {
@ -106,20 +236,27 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
return s.Snapshotter.Mounts(ctx, bkey) return s.Snapshotter.Mounts(ctx, bkey)
} }
func (s *snapshotter) Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return s.createSnapshot(ctx, key, parent, false) return s.createSnapshot(ctx, key, parent, false, opts)
} }
func (s *snapshotter) View(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return s.createSnapshot(ctx, key, parent, true) return s.createSnapshot(ctx, key, parent, true, opts)
} }
func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, readonly bool) ([]mount.Mount, error) { func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, readonly bool, opts []snapshot.Opt) ([]mount.Mount, error) {
ns, err := namespaces.NamespaceRequired(ctx) ns, err := namespaces.NamespaceRequired(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var base snapshot.Info
for _, opt := range opts {
if err := opt(&base); err != nil {
return nil, err
}
}
var m []mount.Mount var m []mount.Mount
if err := update(ctx, s.db, func(tx *bolt.Tx) error { if err := update(ctx, s.db, func(tx *bolt.Tx) error {
bkt, err := createSnapshotterBucket(tx, ns, s.name) bkt, err := createSnapshotterBucket(tx, ns, s.name)
@ -127,24 +264,40 @@ func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, re
return err return err
} }
bkey := string(bkt.Get([]byte(key))) bbkt, err := bkt.CreateBucket([]byte(key))
if bkey != "" { if err != nil {
return errors.Wrapf(errdefs.ErrAlreadyExists, "snapshot %v already exists", key) if err == bolt.ErrBucketExists {
err = errors.Wrapf(errdefs.ErrAlreadyExists, "snapshot %v already exists", key)
}
return err
} }
var bparent string var bparent string
if parent != "" { if parent != "" {
bparent = string(bkt.Get([]byte(parent))) pbkt := bkt.Bucket([]byte(parent))
if bparent == "" { if pbkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", parent) return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", parent)
} }
bparent = string(pbkt.Get(bucketKeyName))
if err := bbkt.Put(bucketKeyParent, []byte(parent)); err != nil {
return err
}
} }
sid, err := bkt.NextSequence() sid, err := bkt.NextSequence()
if err != nil { if err != nil {
return err return err
} }
bkey = createKey(sid, ns, key) bkey := createKey(sid, ns, key)
if err := bkt.Put([]byte(key), []byte(bkey)); err != nil { if err := bbkt.Put(bucketKeyName, []byte(bkey)); err != nil {
return err
}
ts := time.Now().UTC()
if err := boltutil.WriteTimestamps(bbkt, ts, ts); err != nil {
return err
}
if err := boltutil.WriteLabels(bbkt, base.Labels); err != nil {
return err return err
} }
@ -162,37 +315,62 @@ func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, re
return m, nil return m, nil
} }
func (s *snapshotter) Commit(ctx context.Context, name, key string) error { func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error {
ns, err := namespaces.NamespaceRequired(ctx) ns, err := namespaces.NamespaceRequired(ctx)
if err != nil { if err != nil {
return err return err
} }
var base snapshot.Info
for _, opt := range opts {
if err := opt(&base); err != nil {
return err
}
}
return update(ctx, s.db, func(tx *bolt.Tx) error { return update(ctx, s.db, func(tx *bolt.Tx) error {
bkt := getSnapshotterBucket(tx, ns, s.name) bkt := getSnapshotterBucket(tx, ns, s.name)
if bkt == nil { if bkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key) return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key)
} }
nameKey := string(bkt.Get([]byte(name))) bbkt, err := bkt.CreateBucket([]byte(name))
if nameKey != "" { if err != nil {
return errors.Wrapf(errdefs.ErrAlreadyExists, "snapshot %v already exists", name) if err == bolt.ErrBucketExists {
err = errors.Wrapf(errdefs.ErrAlreadyExists, "snapshot %v already exists", name)
}
return err
} }
bkey := string(bkt.Get([]byte(key))) obkt := bkt.Bucket([]byte(key))
if bkey == "" { if obkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key) return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key)
} }
bkey := string(obkt.Get(bucketKeyName))
parent := string(obkt.Get(bucketKeyParent))
sid, err := bkt.NextSequence() sid, err := bkt.NextSequence()
if err != nil { if err != nil {
return err return err
} }
nameKey = createKey(sid, ns, name)
if err := bkt.Put([]byte(name), []byte(nameKey)); err != nil { nameKey := createKey(sid, ns, name)
if err := bbkt.Put(bucketKeyName, []byte(nameKey)); err != nil {
return err return err
} }
if err := bkt.Delete([]byte(key)); err != nil { if err := bbkt.Put(bucketKeyParent, []byte(parent)); err != nil {
return err
}
ts := time.Now().UTC()
if err := boltutil.WriteTimestamps(bbkt, ts, ts); err != nil {
return err
}
if err := boltutil.WriteLabels(bbkt, base.Labels); err != nil {
return err
}
if err := bkt.DeleteBucket([]byte(key)); err != nil {
return err return err
} }
@ -210,16 +388,19 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
} }
return update(ctx, s.db, func(tx *bolt.Tx) error { return update(ctx, s.db, func(tx *bolt.Tx) error {
var bkey string
bkt := getSnapshotterBucket(tx, ns, s.name) bkt := getSnapshotterBucket(tx, ns, s.name)
if bkt == nil { if bkt != nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key) sbkt := bkt.Bucket([]byte(key))
if sbkt != nil {
bkey = string(sbkt.Get(bucketKeyName))
}
} }
bkey := string(bkt.Get([]byte(key)))
if bkey == "" { if bkey == "" {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key) return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key)
} }
if err := bkt.Delete([]byte(key)); err != nil {
if err := bkt.DeleteBucket([]byte(key)); err != nil {
return err return err
} }
@ -227,45 +408,93 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
}) })
} }
type infoPair struct {
bkey string
info snapshot.Info
}
func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error {
ns, err := namespaces.NamespaceRequired(ctx) ns, err := namespaces.NamespaceRequired(ctx)
if err != nil { if err != nil {
return err return err
} }
var keys []string var (
batchSize = 100
pairs = []infoPair{}
lastKey string
)
if err := view(ctx, s.db, func(tx *bolt.Tx) error { for {
bkt := getSnapshotterBucket(tx, ns, s.name) if err := view(ctx, s.db, func(tx *bolt.Tx) error {
if bkt == nil { bkt := getSnapshotterBucket(tx, ns, s.name)
return nil if bkt == nil {
} return nil
bkt.ForEach(func(k, v []byte) error {
if len(v) > 0 {
keys = append(keys, string(v))
} }
c := bkt.Cursor()
var k, v []byte
if lastKey == "" {
k, v = c.First()
} else {
k, v = c.Seek([]byte(lastKey))
}
for k != nil {
if v == nil {
if len(pairs) >= batchSize {
break
}
sbkt := bkt.Bucket(k)
pair := infoPair{
bkey: string(sbkt.Get(bucketKeyName)),
info: snapshot.Info{
Name: string(k),
Parent: string(sbkt.Get(bucketKeyParent)),
},
}
err := boltutil.ReadTimestamps(sbkt, &pair.info.Created, &pair.info.Updated)
if err != nil {
return err
}
pair.info.Labels, err = boltutil.ReadLabels(sbkt)
if err != nil {
return err
}
pairs = append(pairs, pair)
}
k, v = c.Next()
}
lastKey = string(k)
return nil return nil
}) }); err != nil {
return nil
}); err != nil {
return err
}
for _, k := range keys {
info, err := s.Snapshotter.Stat(ctx, k)
if err != nil {
return err return err
} }
info.Name = trimKey(info.Name) for _, pair := range pairs {
if info.Parent != "" { info, err := s.Snapshotter.Stat(ctx, pair.bkey)
info.Parent = trimKey(info.Parent) if err != nil {
return err
}
if err := fn(ctx, overlayInfo(info, pair.info)); err != nil {
return err
}
} }
if err := fn(ctx, info); err != nil {
return err if lastKey == "" {
break
} }
pairs = pairs[:0]
} }
return nil return nil

View File

@ -9,6 +9,7 @@ import (
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
protobuftypes "github.com/gogo/protobuf/types"
) )
// NewSnapshotterFromClient returns a new Snapshotter which communicates // NewSnapshotterFromClient returns a new Snapshotter which communicates
@ -37,6 +38,21 @@ func (r *remoteSnapshotter) Stat(ctx context.Context, key string) (snapshot.Info
return toInfo(resp.Info), nil return toInfo(resp.Info), nil
} }
func (r *remoteSnapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) {
resp, err := r.client.Update(ctx,
&snapshotapi.UpdateSnapshotRequest{
Snapshotter: r.snapshotterName,
Info: fromInfo(info),
UpdateMask: &protobuftypes.FieldMask{
Paths: fieldpaths,
},
})
if err != nil {
return snapshot.Info{}, errdefs.FromGRPC(err)
}
return toInfo(resp.Info), nil
}
func (r *remoteSnapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { func (r *remoteSnapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) {
resp, err := r.client.Usage(ctx, &snapshotapi.UsageRequest{ resp, err := r.client.Usage(ctx, &snapshotapi.UsageRequest{
Snapshotter: r.snapshotterName, Snapshotter: r.snapshotterName,
@ -59,11 +75,18 @@ func (r *remoteSnapshotter) Mounts(ctx context.Context, key string) ([]mount.Mou
return toMounts(resp.Mounts), nil return toMounts(resp.Mounts), nil
} }
func (r *remoteSnapshotter) Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (r *remoteSnapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
var local snapshot.Info
for _, opt := range opts {
if err := opt(&local); err != nil {
return nil, err
}
}
resp, err := r.client.Prepare(ctx, &snapshotapi.PrepareSnapshotRequest{ resp, err := r.client.Prepare(ctx, &snapshotapi.PrepareSnapshotRequest{
Snapshotter: r.snapshotterName, Snapshotter: r.snapshotterName,
Key: key, Key: key,
Parent: parent, Parent: parent,
Labels: local.Labels,
}) })
if err != nil { if err != nil {
return nil, errdefs.FromGRPC(err) return nil, errdefs.FromGRPC(err)
@ -71,11 +94,18 @@ func (r *remoteSnapshotter) Prepare(ctx context.Context, key, parent string) ([]
return toMounts(resp.Mounts), nil return toMounts(resp.Mounts), nil
} }
func (r *remoteSnapshotter) View(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (r *remoteSnapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
var local snapshot.Info
for _, opt := range opts {
if err := opt(&local); err != nil {
return nil, err
}
}
resp, err := r.client.View(ctx, &snapshotapi.ViewSnapshotRequest{ resp, err := r.client.View(ctx, &snapshotapi.ViewSnapshotRequest{
Snapshotter: r.snapshotterName, Snapshotter: r.snapshotterName,
Key: key, Key: key,
Parent: parent, Parent: parent,
Labels: local.Labels,
}) })
if err != nil { if err != nil {
return nil, errdefs.FromGRPC(err) return nil, errdefs.FromGRPC(err)
@ -83,11 +113,18 @@ func (r *remoteSnapshotter) View(ctx context.Context, key, parent string) ([]mou
return toMounts(resp.Mounts), nil return toMounts(resp.Mounts), nil
} }
func (r *remoteSnapshotter) Commit(ctx context.Context, name, key string) error { func (r *remoteSnapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error {
var local snapshot.Info
for _, opt := range opts {
if err := opt(&local); err != nil {
return err
}
}
_, err := r.client.Commit(ctx, &snapshotapi.CommitSnapshotRequest{ _, err := r.client.Commit(ctx, &snapshotapi.CommitSnapshotRequest{
Snapshotter: r.snapshotterName, Snapshotter: r.snapshotterName,
Name: name, Name: name,
Key: key, Key: key,
Labels: local.Labels,
}) })
return errdefs.FromGRPC(err) return errdefs.FromGRPC(err)
} }
@ -130,14 +167,20 @@ func toKind(kind snapshotapi.Kind) snapshot.Kind {
if kind == snapshotapi.KindActive { if kind == snapshotapi.KindActive {
return snapshot.KindActive return snapshot.KindActive
} }
if kind == snapshotapi.KindView {
return snapshot.KindView
}
return snapshot.KindCommitted return snapshot.KindCommitted
} }
func toInfo(info snapshotapi.Info) snapshot.Info { func toInfo(info snapshotapi.Info) snapshot.Info {
return snapshot.Info{ return snapshot.Info{
Name: info.Name, Name: info.Name,
Parent: info.Parent, Parent: info.Parent,
Kind: toKind(info.Kind), Kind: toKind(info.Kind),
Created: info.CreatedAt,
Updated: info.UpdatedAt,
Labels: info.Labels,
} }
} }

View File

@ -97,9 +97,12 @@ func (s *service) Prepare(ctx context.Context, pr *snapshotapi.PrepareSnapshotRe
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Apply namespace
// TODO: Lookup snapshot id from metadata store var opts []snapshot.Opt
mounts, err := sn.Prepare(ctx, pr.Key, pr.Parent) if pr.Labels != nil {
opts = append(opts, snapshot.WithLabels(pr.Labels))
}
mounts, err := sn.Prepare(ctx, pr.Key, pr.Parent, opts...)
if err != nil { if err != nil {
return nil, errdefs.ToGRPC(err) return nil, errdefs.ToGRPC(err)
} }
@ -121,9 +124,11 @@ func (s *service) View(ctx context.Context, pr *snapshotapi.ViewSnapshotRequest)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Apply namespace var opts []snapshot.Opt
// TODO: Lookup snapshot id from metadata store if pr.Labels != nil {
mounts, err := sn.View(ctx, pr.Key, pr.Parent) opts = append(opts, snapshot.WithLabels(pr.Labels))
}
mounts, err := sn.View(ctx, pr.Key, pr.Parent, opts...)
if err != nil { if err != nil {
return nil, errdefs.ToGRPC(err) return nil, errdefs.ToGRPC(err)
} }
@ -138,8 +143,7 @@ func (s *service) Mounts(ctx context.Context, mr *snapshotapi.MountsRequest) (*s
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Apply namespace
// TODO: Lookup snapshot id from metadata store
mounts, err := sn.Mounts(ctx, mr.Key) mounts, err := sn.Mounts(ctx, mr.Key)
if err != nil { if err != nil {
return nil, errdefs.ToGRPC(err) return nil, errdefs.ToGRPC(err)
@ -155,9 +159,12 @@ func (s *service) Commit(ctx context.Context, cr *snapshotapi.CommitSnapshotRequ
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Apply namespace
// TODO: Lookup snapshot id from metadata store var opts []snapshot.Opt
if err := sn.Commit(ctx, cr.Name, cr.Key); err != nil { if cr.Labels != nil {
opts = append(opts, snapshot.WithLabels(cr.Labels))
}
if err := sn.Commit(ctx, cr.Name, cr.Key, opts...); err != nil {
return nil, errdefs.ToGRPC(err) return nil, errdefs.ToGRPC(err)
} }
@ -176,8 +183,7 @@ func (s *service) Remove(ctx context.Context, rr *snapshotapi.RemoveSnapshotRequ
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Apply namespace
// TODO: Lookup snapshot id from metadata store
if err := sn.Remove(ctx, rr.Key); err != nil { if err := sn.Remove(ctx, rr.Key); err != nil {
return nil, errdefs.ToGRPC(err) return nil, errdefs.ToGRPC(err)
} }
@ -196,7 +202,7 @@ func (s *service) Stat(ctx context.Context, sr *snapshotapi.StatSnapshotRequest)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Apply namespace
info, err := sn.Stat(ctx, sr.Key) info, err := sn.Stat(ctx, sr.Key)
if err != nil { if err != nil {
return nil, errdefs.ToGRPC(err) return nil, errdefs.ToGRPC(err)
@ -205,12 +211,27 @@ func (s *service) Stat(ctx context.Context, sr *snapshotapi.StatSnapshotRequest)
return &snapshotapi.StatSnapshotResponse{Info: fromInfo(info)}, nil return &snapshotapi.StatSnapshotResponse{Info: fromInfo(info)}, nil
} }
func (s *service) Update(ctx context.Context, sr *snapshotapi.UpdateSnapshotRequest) (*snapshotapi.UpdateSnapshotResponse, error) {
log.G(ctx).WithField("key", sr.Info.Name).Debugf("Updating snapshot")
sn, err := s.getSnapshotter(sr.Snapshotter)
if err != nil {
return nil, err
}
info, err := sn.Update(ctx, toInfo(sr.Info), sr.UpdateMask.GetPaths()...)
if err != nil {
return nil, errdefs.ToGRPC(err)
}
return &snapshotapi.UpdateSnapshotResponse{Info: fromInfo(info)}, nil
}
func (s *service) List(sr *snapshotapi.ListSnapshotsRequest, ss snapshotapi.Snapshots_ListServer) error { func (s *service) List(sr *snapshotapi.ListSnapshotsRequest, ss snapshotapi.Snapshots_ListServer) error {
sn, err := s.getSnapshotter(sr.Snapshotter) sn, err := s.getSnapshotter(sr.Snapshotter)
if err != nil { if err != nil {
return err return err
} }
// TODO: Apply namespace
var ( var (
buffer []snapshotapi.Info buffer []snapshotapi.Info
sendBlock = func(block []snapshotapi.Info) error { sendBlock = func(block []snapshotapi.Info) error {
@ -250,7 +271,7 @@ func (s *service) Usage(ctx context.Context, ur *snapshotapi.UsageRequest) (*sna
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Apply namespace
usage, err := sn.Usage(ctx, ur.Key) usage, err := sn.Usage(ctx, ur.Key)
if err != nil { if err != nil {
return nil, errdefs.ToGRPC(err) return nil, errdefs.ToGRPC(err)
@ -263,14 +284,20 @@ func fromKind(kind snapshot.Kind) snapshotapi.Kind {
if kind == snapshot.KindActive { if kind == snapshot.KindActive {
return snapshotapi.KindActive return snapshotapi.KindActive
} }
if kind == snapshot.KindView {
return snapshotapi.KindView
}
return snapshotapi.KindCommitted return snapshotapi.KindCommitted
} }
func fromInfo(info snapshot.Info) snapshotapi.Info { func fromInfo(info snapshot.Info) snapshotapi.Info {
return snapshotapi.Info{ return snapshotapi.Info{
Name: info.Name, Name: info.Name,
Parent: info.Parent, Parent: info.Parent,
Kind: fromKind(info.Kind), Kind: fromKind(info.Kind),
CreatedAt: info.Created,
UpdatedAt: info.Updated,
Labels: info.Labels,
} }
} }

View File

@ -103,6 +103,25 @@ func (b *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, erro
return info, nil return info, nil
} }
func (o *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) {
ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil {
return snapshot.Info{}, err
}
info, err = storage.UpdateInfo(ctx, info, fieldpaths...)
if err != nil {
t.Rollback()
return snapshot.Info{}, err
}
if err := t.Commit(); err != nil {
return snapshot.Info{}, err
}
return info, nil
}
// Usage retrieves the disk usage of the top-level snapshot. // Usage retrieves the disk usage of the top-level snapshot.
func (b *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { func (b *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) {
panic("not implemented") panic("not implemented")
@ -129,15 +148,15 @@ func (b *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho
return storage.WalkInfo(ctx, fn) return storage.WalkInfo(ctx, fn)
} }
func (b *snapshotter) Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (b *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return b.makeSnapshot(ctx, snapshot.KindActive, key, parent) return b.makeSnapshot(ctx, snapshot.KindActive, key, parent, opts)
} }
func (b *snapshotter) View(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (b *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return b.makeSnapshot(ctx, snapshot.KindView, key, parent) return b.makeSnapshot(ctx, snapshot.KindView, key, parent, opts)
} }
func (b *snapshotter) makeSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string) ([]mount.Mount, error) { func (b *snapshotter) makeSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, opts []snapshot.Opt) ([]mount.Mount, error) {
ctx, t, err := b.ms.TransactionContext(ctx, true) ctx, t, err := b.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return nil, err return nil, err
@ -150,7 +169,7 @@ func (b *snapshotter) makeSnapshot(ctx context.Context, kind snapshot.Kind, key,
} }
}() }()
s, err := storage.CreateSnapshot(ctx, kind, key, parent) s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -214,7 +233,7 @@ func (b *snapshotter) mounts(dir string) ([]mount.Mount, error) {
}, nil }, nil
} }
func (b *snapshotter) Commit(ctx context.Context, name, key string) (err error) { func (b *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) (err error) {
ctx, t, err := b.ms.TransactionContext(ctx, true) ctx, t, err := b.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return err return err
@ -227,7 +246,7 @@ func (b *snapshotter) Commit(ctx context.Context, name, key string) (err error)
} }
}() }()
id, err := storage.CommitActive(ctx, key, name, snapshot.Usage{}) // TODO(stevvooe): Resolve a usage value for btrfs id, err := storage.CommitActive(ctx, key, name, snapshot.Usage{}, opts...) // TODO(stevvooe): Resolve a usage value for btrfs
if err != nil { if err != nil {
return errors.Wrap(err, "failed to commit") return errors.Wrap(err, "failed to commit")
} }

View File

@ -70,6 +70,25 @@ func (o *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, erro
return info, nil return info, nil
} }
func (o *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) {
ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil {
return snapshot.Info{}, err
}
info, err = storage.UpdateInfo(ctx, info, fieldpaths...)
if err != nil {
t.Rollback()
return snapshot.Info{}, err
}
if err := t.Commit(); err != nil {
return snapshot.Info{}, err
}
return info, nil
}
func (o *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { func (o *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) {
ctx, t, err := o.ms.TransactionContext(ctx, false) ctx, t, err := o.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
@ -93,12 +112,12 @@ func (o *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, er
return usage, nil return usage, nil
} }
func (o *snapshotter) Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return o.createSnapshot(ctx, snapshot.KindActive, key, parent) return o.createSnapshot(ctx, snapshot.KindActive, key, parent, opts)
} }
func (o *snapshotter) View(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return o.createSnapshot(ctx, snapshot.KindView, key, parent) return o.createSnapshot(ctx, snapshot.KindView, key, parent, opts)
} }
// Mounts returns the mounts for the transaction identified by key. Can be // Mounts returns the mounts for the transaction identified by key. Can be
@ -118,7 +137,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
return o.mounts(s), nil return o.mounts(s), nil
} }
func (o *snapshotter) Commit(ctx context.Context, name, key string) error { func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error {
ctx, t, err := o.ms.TransactionContext(ctx, true) ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return err return err
@ -134,7 +153,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string) error {
return err return err
} }
if _, err := storage.CommitActive(ctx, key, name, snapshot.Usage(usage)); err != nil { if _, err := storage.CommitActive(ctx, key, name, snapshot.Usage(usage), opts...); err != nil {
if rerr := t.Rollback(); rerr != nil { if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction")
} }
@ -203,7 +222,7 @@ func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho
return storage.WalkInfo(ctx, fn) return storage.WalkInfo(ctx, fn)
} }
func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string) ([]mount.Mount, error) { func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, opts []snapshot.Opt) ([]mount.Mount, error) {
var ( var (
err error err error
path, td string path, td string
@ -235,7 +254,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, ke
return nil, err return nil, err
} }
s, err := storage.CreateSnapshot(ctx, kind, key, parent) s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...)
if err != nil { if err != nil {
if rerr := t.Rollback(); rerr != nil { if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction")

View File

@ -89,6 +89,25 @@ func (o *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, erro
return info, nil return info, nil
} }
func (o *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) {
ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil {
return snapshot.Info{}, err
}
info, err = storage.UpdateInfo(ctx, info, fieldpaths...)
if err != nil {
t.Rollback()
return snapshot.Info{}, err
}
if err := t.Commit(); err != nil {
return snapshot.Info{}, err
}
return info, nil
}
// Usage returns the resources taken by the snapshot identified by key. // Usage returns the resources taken by the snapshot identified by key.
// //
// For active snapshots, this will scan the usage of the overlay "diff" (aka // For active snapshots, this will scan the usage of the overlay "diff" (aka
@ -121,12 +140,12 @@ func (o *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, er
return usage, nil return usage, nil
} }
func (o *snapshotter) Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return o.createSnapshot(ctx, snapshot.KindActive, key, parent) return o.createSnapshot(ctx, snapshot.KindActive, key, parent, opts)
} }
func (o *snapshotter) View(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
return o.createSnapshot(ctx, snapshot.KindView, key, parent) return o.createSnapshot(ctx, snapshot.KindView, key, parent, opts)
} }
// Mounts returns the mounts for the transaction identified by key. Can be // Mounts returns the mounts for the transaction identified by key. Can be
@ -146,7 +165,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
return o.mounts(s), nil return o.mounts(s), nil
} }
func (o *snapshotter) Commit(ctx context.Context, name, key string) error { func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error {
ctx, t, err := o.ms.TransactionContext(ctx, true) ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return err return err
@ -171,7 +190,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string) error {
return err return err
} }
if _, err = storage.CommitActive(ctx, key, name, snapshot.Usage(usage)); err != nil { if _, err = storage.CommitActive(ctx, key, name, snapshot.Usage(usage), opts...); err != nil {
return errors.Wrap(err, "failed to commit snapshot") return errors.Wrap(err, "failed to commit snapshot")
} }
return t.Commit() return t.Commit()
@ -230,7 +249,7 @@ func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho
return storage.WalkInfo(ctx, fn) return storage.WalkInfo(ctx, fn)
} }
func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string) ([]mount.Mount, error) { func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, opts []snapshot.Opt) ([]mount.Mount, error) {
var ( var (
path string path string
snapshotDir = filepath.Join(o.root, "snapshots") snapshotDir = filepath.Join(o.root, "snapshots")
@ -270,7 +289,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, ke
return nil, err return nil, err
} }
s, err := storage.CreateSnapshot(ctx, kind, key, parent) s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...)
if err != nil { if err != nil {
if rerr := t.Rollback(); rerr != nil { if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction")

View File

@ -2,12 +2,13 @@ package snapshot
import ( import (
"context" "context"
"time"
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
) )
// Kind identifies the kind of snapshot. // Kind identifies the kind of snapshot.
type Kind int type Kind uint8
// definitions of snapshot kinds // definitions of snapshot kinds
const ( const (
@ -31,9 +32,12 @@ func (k Kind) String() string {
// Info provides information about a particular snapshot. // Info provides information about a particular snapshot.
type Info struct { type Info struct {
Kind Kind // active or committed snapshot Kind Kind // active or committed snapshot
Name string // name or key of snapshot Name string // name or key of snapshot
Parent string // name of parent snapshot Parent string // name of parent snapshot
Labels map[string]string // Labels for snapshot
Created time.Time // Created time
Updated time.Time // Last update time
} }
// Usage defines statistics for disk resources consumed by the snapshot. // Usage defines statistics for disk resources consumed by the snapshot.
@ -177,6 +181,11 @@ type Snapshotter interface {
// the kind of snapshot. // the kind of snapshot.
Stat(ctx context.Context, key string) (Info, error) Stat(ctx context.Context, key string) (Info, error)
// Update updates the infor for a snapshot.
//
// Only mutable properties of a snapshot may be updated.
Update(ctx context.Context, info Info, fieldpaths ...string) (Info, error)
// Usage returns the resource usage of an active or committed snapshot // Usage returns the resource usage of an active or committed snapshot
// excluding the usage of parent snapshots. // excluding the usage of parent snapshots.
// //
@ -208,7 +217,7 @@ type Snapshotter interface {
// one is done with the transaction, Remove should be called on the key. // one is done with the transaction, Remove should be called on the key.
// //
// Multiple calls to Prepare or View with the same key should fail. // Multiple calls to Prepare or View with the same key should fail.
Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) Prepare(ctx context.Context, key, parent string, opts ...Opt) ([]mount.Mount, error)
// View behaves identically to Prepare except the result may not be // View behaves identically to Prepare except the result may not be
// committed back to the snapshot snapshotter. View returns a readonly view on // committed back to the snapshot snapshotter. View returns a readonly view on
@ -223,7 +232,7 @@ type Snapshotter interface {
// Commit may not be called on the provided key and will return an error. // Commit may not be called on the provided key and will return an error.
// To collect the resources associated with key, Remove must be called with // To collect the resources associated with key, Remove must be called with
// key as the argument. // key as the argument.
View(ctx context.Context, key, parent string) ([]mount.Mount, error) View(ctx context.Context, key, parent string, opts ...Opt) ([]mount.Mount, error)
// Commit captures the changes between key and its parent into a snapshot // Commit captures the changes between key and its parent into a snapshot
// identified by name. The name can then be used with the snapshotter's other // identified by name. The name can then be used with the snapshotter's other
@ -235,7 +244,7 @@ type Snapshotter interface {
// Commit may be called multiple times on the same key. Snapshots created // Commit may be called multiple times on the same key. Snapshots created
// in this manner will all reference the parent used to start the // in this manner will all reference the parent used to start the
// transaction. // transaction.
Commit(ctx context.Context, name, key string) error Commit(ctx context.Context, name, key string, opts ...Opt) error
// Remove the committed or active snapshot by the provided key. // Remove the committed or active snapshot by the provided key.
// //
@ -249,3 +258,14 @@ type Snapshotter interface {
// snapshotter, the function will be called. // snapshotter, the function will be called.
Walk(ctx context.Context, fn func(context.Context, Info) error) error Walk(ctx context.Context, fn func(context.Context, Info) error) error
} }
// Opt allows setting mutable snapshot properties on creation
type Opt func(info *Info) error
// WithLabels adds labels to a created snapshot
func WithLabels(labels map[string]string) Opt {
return func(info *Info) error {
info.Labels = labels
return nil
}
}

View File

@ -4,12 +4,13 @@ import (
"context" "context"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"strings"
"time"
"github.com/boltdb/bolt" "github.com/boltdb/bolt"
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/metadata/boltutil"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
db "github.com/containerd/containerd/snapshot/storage/proto"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -18,6 +19,12 @@ var (
bucketKeySnapshot = []byte("snapshots") bucketKeySnapshot = []byte("snapshots")
bucketKeyParents = []byte("parents") bucketKeyParents = []byte("parents")
bucketKeyID = []byte("id")
bucketKeyParent = []byte("parent")
bucketKeyKind = []byte("kind")
bucketKeyInodes = []byte("inodes")
bucketKeySize = []byte("size")
// ErrNoTransaction is returned when an operation is attempted with // ErrNoTransaction is returned when an operation is attempted with
// a context which is not inside of a transaction. // a context which is not inside of a transaction.
ErrNoTransaction = errors.New("no transaction in context") ErrNoTransaction = errors.New("no transaction in context")
@ -65,24 +72,75 @@ func getParentPrefix(b []byte) uint64 {
// GetInfo returns the snapshot Info directly from the metadata. Requires a // GetInfo returns the snapshot Info directly from the metadata. Requires a
// context with a storage transaction. // context with a storage transaction.
func GetInfo(ctx context.Context, key string) (string, snapshot.Info, snapshot.Usage, error) { func GetInfo(ctx context.Context, key string) (string, snapshot.Info, snapshot.Usage, error) {
var ss db.Snapshot var (
err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { id uint64
return getSnapshot(bkt, key, &ss) su snapshot.Usage
si = snapshot.Info{
Name: key,
}
)
err := withSnapshotBucket(ctx, key, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
getUsage(bkt, &su)
return readSnapshot(bkt, &id, &si)
}) })
if err != nil { if err != nil {
return "", snapshot.Info{}, snapshot.Usage{}, err return "", snapshot.Info{}, snapshot.Usage{}, err
} }
usage := snapshot.Usage{ return fmt.Sprintf("%d", id), si, su, nil
Inodes: ss.Inodes, }
Size: ss.Size_,
}
return fmt.Sprint(ss.ID), snapshot.Info{ func UpdateInfo(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) {
Name: key, updated := snapshot.Info{
Parent: ss.Parent, Name: info.Name,
Kind: snapshot.Kind(ss.Kind), }
}, usage, nil err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
sbkt := bkt.Bucket([]byte(info.Name))
if sbkt == nil {
return errors.Wrap(errdefs.ErrNotFound, "snapshot does not exist")
}
if err := readSnapshot(sbkt, nil, &updated); err != nil {
return err
}
if len(fieldpaths) > 0 {
for _, path := range fieldpaths {
if strings.HasPrefix(path, "labels.") {
if updated.Labels == nil {
updated.Labels = map[string]string{}
}
key := strings.TrimPrefix(path, "labels.")
updated.Labels[key] = info.Labels[key]
continue
}
switch path {
case "labels":
updated.Labels = info.Labels
default:
return errors.Wrapf(errdefs.ErrInvalidArgument, "cannot update %q field on snapshot %q", path, info.Name)
}
}
} else {
// Set mutable fields
updated.Labels = info.Labels
}
updated.Updated = time.Now().UTC()
if err := boltutil.WriteTimestamps(sbkt, updated.Created, updated.Updated); err != nil {
return err
}
if err := boltutil.WriteLabels(sbkt, updated.Labels); err != nil {
return err
}
return nil
})
if err != nil {
return snapshot.Info{}, err
}
return updated, nil
} }
// WalkInfo iterates through all metadata Info for the stored snapshots and // WalkInfo iterates through all metadata Info for the stored snapshots and
@ -91,21 +149,21 @@ func GetInfo(ctx context.Context, key string) (string, snapshot.Info, snapshot.U
func WalkInfo(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { func WalkInfo(ctx context.Context, fn func(context.Context, snapshot.Info) error) error {
return withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { return withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
return bkt.ForEach(func(k, v []byte) error { return bkt.ForEach(func(k, v []byte) error {
// skip nested buckets // skip non buckets
if v == nil { if v != nil {
return nil return nil
} }
var ss db.Snapshot var (
if err := proto.Unmarshal(v, &ss); err != nil { sbkt = bkt.Bucket(k)
return errors.Wrap(err, "failed to unmarshal snapshot") si = snapshot.Info{
Name: string(k),
}
)
if err := readSnapshot(sbkt, nil, &si); err != nil {
return err
} }
info := snapshot.Info{ return fn(ctx, si)
Name: string(k),
Parent: ss.Parent,
Kind: snapshot.Kind(ss.Kind),
}
return fn(ctx, info)
}) })
}) })
} }
@ -114,30 +172,25 @@ func WalkInfo(ctx context.Context, fn func(context.Context, snapshot.Info) error
// referenced by the given key. Requires a context with a storage transaction. // referenced by the given key. Requires a context with a storage transaction.
func GetSnapshot(ctx context.Context, key string) (s Snapshot, err error) { func GetSnapshot(ctx context.Context, key string) (s Snapshot, err error) {
err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
b := bkt.Get([]byte(key)) sbkt := bkt.Bucket([]byte(key))
if len(b) == 0 { if sbkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v", key) return errors.Wrap(errdefs.ErrNotFound, "snapshot does not exist")
} }
var ss db.Snapshot s.ID = fmt.Sprintf("%d", readID(sbkt))
if err := proto.Unmarshal(b, &ss); err != nil { s.Kind = readKind(sbkt)
return errors.Wrap(err, "failed to unmarshal snapshot")
}
if ss.Kind != db.KindActive && ss.Kind != db.KindView { if s.Kind != snapshot.KindActive && s.Kind != snapshot.KindView {
return errors.Wrapf(errdefs.ErrFailedPrecondition, "requested snapshot %v not active or view", key) return errors.Wrapf(errdefs.ErrFailedPrecondition, "requested snapshot %v not active or view", key)
} }
s.ID = fmt.Sprintf("%d", ss.ID) if parentKey := sbkt.Get(bucketKeyParent); len(parentKey) > 0 {
s.Kind = snapshot.Kind(ss.Kind) spbkt := bkt.Bucket(parentKey)
if spbkt == nil {
if ss.Parent != "" { return errors.Wrap(errdefs.ErrNotFound, "parent does not exist")
var parent db.Snapshot
if err := getSnapshot(bkt, ss.Parent, &parent); err != nil {
return errors.Wrap(err, "failed to get parent snapshot")
} }
s.ParentIDs, err = parents(bkt, &parent) s.ParentIDs, err = parents(bkt, spbkt, readID(spbkt))
if err != nil { if err != nil {
return errors.Wrap(err, "failed to get parent chain") return errors.Wrap(err, "failed to get parent chain")
} }
@ -152,30 +205,39 @@ func GetSnapshot(ctx context.Context, key string) (s Snapshot, err error) {
} }
// CreateSnapshot inserts a record for an active or view snapshot with the provided parent. // CreateSnapshot inserts a record for an active or view snapshot with the provided parent.
func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string) (s Snapshot, err error) { func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, opts ...snapshot.Opt) (s Snapshot, err error) {
switch kind { switch kind {
case snapshot.KindActive, snapshot.KindView: case snapshot.KindActive, snapshot.KindView:
default: default:
return Snapshot{}, errors.Wrapf(errdefs.ErrInvalidArgument, "snapshot type %v invalid; only snapshots of type Active or View can be created", kind) return Snapshot{}, errors.Wrapf(errdefs.ErrInvalidArgument, "snapshot type %v invalid; only snapshots of type Active or View can be created", kind)
} }
var base snapshot.Info
for _, opt := range opts {
if err := opt(&base); err != nil {
return Snapshot{}, err
}
}
err = createBucketIfNotExists(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { err = createBucketIfNotExists(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
var ( var (
parentS *db.Snapshot spbkt *bolt.Bucket
) )
if parent != "" { if parent != "" {
parentS = new(db.Snapshot) spbkt = bkt.Bucket([]byte(parent))
if err := getSnapshot(bkt, parent, parentS); err != nil { if spbkt == nil {
return errors.Wrap(err, "failed to get parent snapshot") return errors.Wrap(errdefs.ErrNotFound, "missing parent bucket")
} }
if parentS.Kind != db.KindCommitted { if readKind(spbkt) != snapshot.KindCommitted {
return errors.Wrap(errdefs.ErrInvalidArgument, "parent is not committed snapshot") return errors.Wrap(errdefs.ErrInvalidArgument, "parent is not committed snapshot")
} }
} }
b := bkt.Get([]byte(key)) sbkt, err := bkt.CreateBucket([]byte(key))
if len(b) != 0 { if err != nil {
return errors.Wrapf(errdefs.ErrAlreadyExists, "snapshot %v", key) if err == bolt.ErrBucketExists {
err = errors.Wrapf(errdefs.ErrAlreadyExists, "snapshot %v", key)
}
return err
} }
id, err := bkt.NextSequence() id, err := bkt.NextSequence()
@ -183,23 +245,28 @@ func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string)
return errors.Wrap(err, "unable to get identifier") return errors.Wrap(err, "unable to get identifier")
} }
ss := db.Snapshot{ t := time.Now().UTC()
ID: id, si := snapshot.Info{
Parent: parent, Parent: parent,
Kind: db.Kind(kind), Kind: kind,
Labels: base.Labels,
Created: t,
Updated: t,
} }
if err := putSnapshot(bkt, key, &ss); err != nil { if err := putSnapshot(sbkt, id, si); err != nil {
return err return err
} }
if parentS != nil { if spbkt != nil {
pid := readID(spbkt)
// Store a backlink from the key to the parent. Store the snapshot name // Store a backlink from the key to the parent. Store the snapshot name
// as the value to allow following the backlink to the snapshot value. // as the value to allow following the backlink to the snapshot value.
if err := pbkt.Put(parentKey(parentS.ID, ss.ID), []byte(key)); err != nil { if err := pbkt.Put(parentKey(pid, id), []byte(key)); err != nil {
return errors.Wrap(err, "failed to write parent link") return errors.Wrap(err, "failed to write parent link")
} }
s.ParentIDs, err = parents(bkt, parentS) s.ParentIDs, err = parents(bkt, spbkt, pid)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to get parent chain") return errors.Wrap(err, "failed to get parent chain")
} }
@ -219,47 +286,50 @@ func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string)
// Remove removes a snapshot from the metastore. The string identifier for the // Remove removes a snapshot from the metastore. The string identifier for the
// snapshot is returned as well as the kind. The provided context must contain a // snapshot is returned as well as the kind. The provided context must contain a
// writable transaction. // writable transaction.
func Remove(ctx context.Context, key string) (id string, k snapshot.Kind, err error) { func Remove(ctx context.Context, key string) (string, snapshot.Kind, error) {
err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { var (
var ss db.Snapshot id uint64
b := bkt.Get([]byte(key)) si snapshot.Info
if len(b) == 0 { )
if err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
sbkt := bkt.Bucket([]byte(key))
if sbkt == nil {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v", key) return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v", key)
} }
if err := proto.Unmarshal(b, &ss); err != nil { if err := readSnapshot(sbkt, &id, &si); err != nil {
return errors.Wrap(err, "failed to unmarshal snapshot") errors.Wrapf(err, "failed to read snapshot %s", key)
} }
if pbkt != nil { if pbkt != nil {
k, _ := pbkt.Cursor().Seek(parentPrefixKey(ss.ID)) k, _ := pbkt.Cursor().Seek(parentPrefixKey(id))
if getParentPrefix(k) == ss.ID { if getParentPrefix(k) == id {
return errors.Errorf("cannot remove snapshot with child") return errors.Errorf("cannot remove snapshot with child")
} }
if ss.Parent != "" { if si.Parent != "" {
var ps db.Snapshot spbkt := bkt.Bucket([]byte(si.Parent))
if err := getSnapshot(bkt, ss.Parent, &ps); err != nil { if spbkt == nil {
return errors.Wrap(err, "failed to get parent snapshot") return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v", key)
} }
if err := pbkt.Delete(parentKey(ps.ID, ss.ID)); err != nil { if err := pbkt.Delete(parentKey(readID(spbkt), id)); err != nil {
return errors.Wrap(err, "failed to delte parent link") return errors.Wrap(err, "failed to delete parent link")
} }
} }
} }
if err := bkt.Delete([]byte(key)); err != nil { if err := bkt.DeleteBucket([]byte(key)); err != nil {
return errors.Wrap(err, "failed to delete snapshot") return errors.Wrap(err, "failed to delete snapshot")
} }
id = fmt.Sprintf("%d", ss.ID)
k = snapshot.Kind(ss.Kind)
return nil return nil
}) }); err != nil {
return "", 0, err
}
return return fmt.Sprintf("%d", id), si.Kind, nil
} }
// CommitActive renames the active snapshot transaction referenced by `key` // CommitActive renames the active snapshot transaction referenced by `key`
@ -268,52 +338,94 @@ func Remove(ctx context.Context, key string) (id string, k snapshot.Kind, err er
// lookup or removal. The returned string identifier for the committed snapshot // lookup or removal. The returned string identifier for the committed snapshot
// is the same identifier of the original active snapshot. The provided context // is the same identifier of the original active snapshot. The provided context
// must contain a writable transaction. // must contain a writable transaction.
func CommitActive(ctx context.Context, key, name string, usage snapshot.Usage) (id string, err error) { func CommitActive(ctx context.Context, key, name string, usage snapshot.Usage, opts ...snapshot.Opt) (string, error) {
err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { var (
b := bkt.Get([]byte(name)) id uint64
if len(b) != 0 { base snapshot.Info
return errors.Wrapf(errdefs.ErrAlreadyExists, "committed snapshot %v", name) )
for _, opt := range opts {
if err := opt(&base); err != nil {
return "", err
}
}
if err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
dbkt, err := bkt.CreateBucket([]byte(name))
if err != nil {
if err == bolt.ErrBucketExists {
err = errdefs.ErrAlreadyExists
}
return errors.Wrapf(err, "committed snapshot %v", name)
}
sbkt := bkt.Bucket([]byte(key))
if sbkt == nil {
return errors.Wrap(errdefs.ErrNotFound, "failed to get active snapshot")
} }
var ss db.Snapshot var si snapshot.Info
if err := getSnapshot(bkt, key, &ss); err != nil { if err := readSnapshot(sbkt, &id, &si); err != nil {
return errors.Wrap(err, "failed to get active snapshot") return errors.Wrap(err, "failed to read snapshot")
} }
if ss.Kind != db.KindActive {
if si.Kind != snapshot.KindActive {
return errors.Wrapf(errdefs.ErrFailedPrecondition, "snapshot %v is not active", name) return errors.Wrapf(errdefs.ErrFailedPrecondition, "snapshot %v is not active", name)
} }
si.Kind = snapshot.KindCommitted
si.Created = time.Now().UTC()
si.Updated = si.Created
ss.Kind = db.KindCommitted // Replace labels, do not inherit
ss.Inodes = usage.Inodes si.Labels = base.Labels
ss.Size_ = usage.Size
if err := putSnapshot(bkt, name, &ss); err != nil { if err := putSnapshot(dbkt, id, si); err != nil {
return err return err
} }
if err := bkt.Delete([]byte(key)); err != nil { if err := putUsage(dbkt, usage); err != nil {
return err
}
if err := bkt.DeleteBucket([]byte(key)); err != nil {
return errors.Wrap(err, "failed to delete active") return errors.Wrap(err, "failed to delete active")
} }
if ss.Parent != "" { if si.Parent != "" {
var ps db.Snapshot spbkt := bkt.Bucket([]byte(si.Parent))
if err := getSnapshot(bkt, ss.Parent, &ps); err != nil { if spbkt == nil {
return errors.Wrap(err, "failed to get parent snapshot") return errors.Wrap(errdefs.ErrNotFound, "missing parent")
} }
pid := readID(spbkt)
// Updates parent back link to use new key // Updates parent back link to use new key
if err := pbkt.Put(parentKey(ps.ID, ss.ID), []byte(name)); err != nil { if err := pbkt.Put(parentKey(pid, id), []byte(name)); err != nil {
return errors.Wrap(err, "failed to update parent link") return errors.Wrap(err, "failed to update parent link")
} }
} }
id = fmt.Sprintf("%d", ss.ID)
return nil return nil
}) }); err != nil {
if err != nil {
return "", err return "", err
} }
return return fmt.Sprintf("%d", id), nil
}
func withSnapshotBucket(ctx context.Context, key string, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
t, ok := ctx.Value(transactionKey{}).(*boltFileTransactor)
if !ok {
return ErrNoTransaction
}
bkt := t.tx.Bucket(bucketKeyStorageVersion)
if bkt == nil {
return errors.Wrap(errdefs.ErrNotFound, "bucket does not exist")
}
bkt = bkt.Bucket(bucketKeySnapshot)
if bkt == nil {
return errors.Wrap(errdefs.ErrNotFound, "snapshots bucket does not exist")
}
bkt = bkt.Bucket([]byte(key))
if bkt == nil {
return errors.Wrap(errdefs.ErrNotFound, "snapshot does not exist")
}
return fn(ctx, bkt, bkt.Bucket(bucketKeyParents))
} }
func withBucket(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error { func withBucket(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
@ -349,41 +461,133 @@ func createBucketIfNotExists(ctx context.Context, fn func(context.Context, *bolt
return fn(ctx, sbkt, pbkt) return fn(ctx, sbkt, pbkt)
} }
func parents(bkt *bolt.Bucket, parent *db.Snapshot) (parents []string, err error) { func parents(bkt, pbkt *bolt.Bucket, parent uint64) (parents []string, err error) {
for { for {
parents = append(parents, fmt.Sprintf("%d", parent.ID)) parents = append(parents, fmt.Sprintf("%d", parent))
if parent.Parent == "" { parentKey := pbkt.Get(bucketKeyParent)
if len(parentKey) == 0 {
return return
} }
pbkt = bkt.Bucket(parentKey)
var ps db.Snapshot if pbkt == nil {
if err := getSnapshot(bkt, parent.Parent, &ps); err != nil { return nil, errors.Wrap(errdefs.ErrNotFound, "missing parent")
return nil, errors.Wrap(err, "failed to get parent snapshot")
} }
parent = &ps
parent = readID(pbkt)
} }
} }
func getSnapshot(bkt *bolt.Bucket, key string, ss *db.Snapshot) error { func readKind(bkt *bolt.Bucket) (k snapshot.Kind) {
b := bkt.Get([]byte(key)) kind := bkt.Get(bucketKeyKind)
if len(b) == 0 { if len(kind) == 1 {
return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v", key) k = snapshot.Kind(kind[0])
} }
if err := proto.Unmarshal(b, ss); err != nil { return
return errors.Wrap(err, "failed to unmarshal snapshot") }
func readID(bkt *bolt.Bucket) uint64 {
id, _ := binary.Uvarint(bkt.Get(bucketKeyID))
return id
}
func readSnapshot(bkt *bolt.Bucket, id *uint64, si *snapshot.Info) error {
if id != nil {
*id = readID(bkt)
} }
if si != nil {
si.Kind = readKind(bkt)
si.Parent = string(bkt.Get(bucketKeyParent))
if err := boltutil.ReadTimestamps(bkt, &si.Created, &si.Updated); err != nil {
return err
}
labels, err := boltutil.ReadLabels(bkt)
if err != nil {
return err
}
si.Labels = labels
}
return nil return nil
} }
func putSnapshot(bkt *bolt.Bucket, key string, ss *db.Snapshot) error { func putSnapshot(bkt *bolt.Bucket, id uint64, si snapshot.Info) error {
b, err := proto.Marshal(ss) idEncoded, err := encodeID(id)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to marshal snapshot") return err
} }
if err := bkt.Put([]byte(key), b); err != nil { updates := [][2][]byte{
return errors.Wrap(err, "failed to save snapshot") {bucketKeyID, idEncoded},
{bucketKeyKind, []byte{byte(si.Kind)}},
}
if si.Parent != "" {
updates = append(updates, [2][]byte{bucketKeyParent, []byte(si.Parent)})
}
for _, v := range updates {
if err := bkt.Put(v[0], v[1]); err != nil {
return err
}
}
if err := boltutil.WriteTimestamps(bkt, si.Created, si.Updated); err != nil {
return err
}
if err := boltutil.WriteLabels(bkt, si.Labels); err != nil {
return err
}
return nil
}
func getUsage(bkt *bolt.Bucket, usage *snapshot.Usage) {
usage.Inodes, _ = binary.Varint(bkt.Get(bucketKeyInodes))
usage.Size, _ = binary.Varint(bkt.Get(bucketKeySize))
}
func putUsage(bkt *bolt.Bucket, usage snapshot.Usage) error {
for _, v := range []struct {
key []byte
value int64
}{
{bucketKeyInodes, usage.Inodes},
{bucketKeySize, usage.Size},
} {
e, err := encodeSize(v.value)
if err != nil {
return err
}
if err := bkt.Put(v.key, e); err != nil {
return err
}
} }
return nil return nil
} }
func encodeSize(size int64) ([]byte, error) {
var (
buf [binary.MaxVarintLen64]byte
sizeEncoded []byte = buf[:]
)
sizeEncoded = sizeEncoded[:binary.PutVarint(sizeEncoded, size)]
if len(sizeEncoded) == 0 {
return nil, fmt.Errorf("failed encoding size = %v", size)
}
return sizeEncoded, nil
}
func encodeID(id uint64) ([]byte, error) {
var (
buf [binary.MaxVarintLen64]byte
idEncoded []byte = buf[:]
)
idEncoded = idEncoded[:binary.PutUvarint(idEncoded, id)]
if len(idEncoded) == 0 {
return nil, fmt.Errorf("failed encoding id = %v", id)
}
return idEncoded, nil
}

View File

@ -5,8 +5,7 @@ import (
"testing" "testing"
// Does not require root but flag must be defined for snapshot tests // Does not require root but flag must be defined for snapshot tests
"github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/storage/proto"
_ "github.com/containerd/containerd/testutil" _ "github.com/containerd/containerd/testutil"
) )
@ -16,34 +15,6 @@ func TestMetastore(t *testing.T) {
}) })
} }
// TestKidnConversion ensures we can blindly cast from protobuf types.
func TestKindConversion(t *testing.T) {
for _, testcase := range []struct {
s snapshot.Kind
p proto.Kind
}{
{
s: snapshot.KindView,
p: proto.KindView,
},
{
s: snapshot.KindActive,
p: proto.KindActive,
},
{
s: snapshot.KindCommitted,
p: proto.KindCommitted,
},
} {
if testcase.s != snapshot.Kind(testcase.p) {
t.Fatalf("snapshot kind value cast failed: %v != %v", testcase.s, testcase.p)
}
if testcase.p != proto.Kind(testcase.s) {
t.Fatalf("proto kind value cast failed: %v != %v", testcase.s, testcase.p)
}
}
}
func BenchmarkSuite(b *testing.B) { func BenchmarkSuite(b *testing.B) {
Benchmarks(b, "BoltDBBench", func(root string) (*MetaStore, error) { Benchmarks(b, "BoltDBBench", func(root string) (*MetaStore, error) {
return NewMetaStore(filepath.Join(root, "metadata.db")) return NewMetaStore(filepath.Join(root, "metadata.db"))

View File

@ -2,9 +2,11 @@ package storage
import ( import (
"context" "context"
"fmt"
"io/ioutil" "io/ioutil"
"os" "os"
"testing" "testing"
"time"
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
@ -40,6 +42,7 @@ func MetaStoreSuite(t *testing.T, name string, meta func(root string) (*MetaStor
t.Run("Remove", makeTest(t, name, meta, inWriteTransaction(testRemove))) t.Run("Remove", makeTest(t, name, meta, inWriteTransaction(testRemove)))
t.Run("RemoveNotExist", makeTest(t, name, meta, inWriteTransaction(testRemoveNotExist))) t.Run("RemoveNotExist", makeTest(t, name, meta, inWriteTransaction(testRemoveNotExist)))
t.Run("RemoveWithChildren", makeTest(t, name, meta, inWriteTransaction(testRemoveWithChildren))) t.Run("RemoveWithChildren", makeTest(t, name, meta, inWriteTransaction(testRemoveWithChildren)))
t.Run("ParentIDs", makeTest(t, name, meta, inWriteTransaction(testParents)))
} }
// makeTest creates a testsuite with a writable transaction // makeTest creates a testsuite with a writable transaction
@ -236,6 +239,9 @@ func testGetInfo(ctx context.Context, t *testing.T, ms *MetaStore) {
if err != nil { if err != nil {
t.Fatalf("GetInfo on %v failed: %+v", key, err) t.Fatalf("GetInfo on %v failed: %+v", key, err)
} }
// TODO: Check timestamp range
info.Created = time.Time{}
info.Updated = time.Time{}
assert.Equal(t, expected, info) assert.Equal(t, expected, info)
} }
} }
@ -251,6 +257,9 @@ func testWalk(ctx context.Context, t *testing.T, ms *MetaStore) {
if _, ok := found[info.Name]; ok { if _, ok := found[info.Name]; ok {
return errors.Errorf("entry already encountered") return errors.Errorf("entry already encountered")
} }
// TODO: Check time range
info.Created = time.Time{}
info.Updated = time.Time{}
found[info.Name] = info found[info.Name] = info
return nil return nil
}) })
@ -545,3 +554,86 @@ func testRemoveNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, _, err := Remove(ctx, "does-not-exist") _, _, err := Remove(ctx, "does-not-exist")
assertNotExist(t, err) assertNotExist(t, err)
} }
func testParents(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err)
}
testcases := []struct {
Name string
Parents int
}{
{"committed-1", 0},
{"committed-2", 1},
{"active-1", 0},
{"active-2", 1},
{"active-3", 2},
{"view-1", 0},
{"view-2", 2},
}
for _, tc := range testcases {
name := tc.Name
expectedID := ""
expectedParents := []string{}
for i := tc.Parents; i >= 0; i-- {
sid, info, _, err := GetInfo(ctx, name)
if err != nil {
t.Fatalf("Failed to get snapshot %s: %v", tc.Name, err)
}
var (
id string
parents []string
)
if info.Kind == snapshot.KindCommitted {
// When commited, create view and resolve from view
nid := fmt.Sprintf("test-%s-%d", tc.Name, i)
s, err := CreateSnapshot(ctx, snapshot.KindView, nid, name)
if err != nil {
t.Fatalf("Failed to get snapshot %s: %v", tc.Name, err)
}
if len(s.ParentIDs) != i+1 {
t.Fatalf("Unexpected number of parents for view of %s: %d, expected %d", name, len(s.ParentIDs), i+1)
}
id = s.ParentIDs[0]
parents = s.ParentIDs[1:]
} else {
s, err := GetSnapshot(ctx, name)
if err != nil {
t.Fatalf("Failed to get snapshot %s: %v", tc.Name, err)
}
if len(s.ParentIDs) != i {
t.Fatalf("Unexpected number of parents for %s: %d, expected %d", name, len(s.ParentIDs), i)
}
id = s.ID
parents = s.ParentIDs
}
if sid != id {
t.Fatalf("Info ID mismatched resolved snapshot ID for %s, %s vs %s", name, sid, id)
}
if expectedID != "" {
if id != expectedID {
t.Errorf("Unexpected ID of parent: %s, expected %s", id, expectedID)
}
}
if len(expectedParents) > 0 {
for j := range expectedParents {
if parents[j] != expectedParents[j] {
t.Errorf("Unexpected ID in parent array at %d: %s, expected %s", j, parents[j], expectedParents[j])
}
}
}
if i > 0 {
name = info.Parent
expectedID = parents[0]
expectedParents = parents[1:]
}
}
}
}

View File

@ -1,506 +0,0 @@
// Code generated by protoc-gen-gogo.
// source: github.com/containerd/containerd/snapshot/storage/proto/record.proto
// DO NOT EDIT!
/*
Package proto is a generated protocol buffer package.
It is generated from these files:
github.com/containerd/containerd/snapshot/storage/proto/record.proto
It has these top-level messages:
Snapshot
*/
package proto
import proto1 "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import strings "strings"
import reflect "reflect"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto1.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto1.GoGoProtoPackageIsVersion2 // please upgrade the proto package
type Kind int32
const (
KindUnknown Kind = 0
KindView Kind = 1
KindActive Kind = 2
KindCommitted Kind = 3
)
var Kind_name = map[int32]string{
0: "UNKNOWN",
1: "VIEW",
2: "ACTIVE",
3: "COMMITTED",
}
var Kind_value = map[string]int32{
"UNKNOWN": 0,
"VIEW": 1,
"ACTIVE": 2,
"COMMITTED": 3,
}
func (x Kind) String() string {
return proto1.EnumName(Kind_name, int32(x))
}
func (Kind) EnumDescriptor() ([]byte, []int) { return fileDescriptorRecord, []int{0} }
// Snapshot defines the storage type for a snapshot in the
// metadata store.
type Snapshot struct {
ID uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
Parent string `protobuf:"bytes,2,opt,name=parent,proto3" json:"parent,omitempty"`
Kind Kind `protobuf:"varint,4,opt,name=kind,proto3,enum=containerd.snapshot.v1.Kind" json:"kind,omitempty"`
// inodes stores the number inodes in use for the snapshot.
//
// Only valid for committed snapshots.
Inodes int64 `protobuf:"varint,6,opt,name=inodes,proto3" json:"inodes,omitempty"`
// Size reports the disk used by the snapshot, excluding the parents.
//
// Only valid for committed snapshots, active snapshots must read the
// current usage from the disk.
Size_ int64 `protobuf:"varint,7,opt,name=size,proto3" json:"size,omitempty"`
}
func (m *Snapshot) Reset() { *m = Snapshot{} }
func (*Snapshot) ProtoMessage() {}
func (*Snapshot) Descriptor() ([]byte, []int) { return fileDescriptorRecord, []int{0} }
func init() {
proto1.RegisterType((*Snapshot)(nil), "containerd.snapshot.v1.Snapshot")
proto1.RegisterEnum("containerd.snapshot.v1.Kind", Kind_name, Kind_value)
}
func (m *Snapshot) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Snapshot) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.ID != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintRecord(dAtA, i, uint64(m.ID))
}
if len(m.Parent) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintRecord(dAtA, i, uint64(len(m.Parent)))
i += copy(dAtA[i:], m.Parent)
}
if m.Kind != 0 {
dAtA[i] = 0x20
i++
i = encodeVarintRecord(dAtA, i, uint64(m.Kind))
}
if m.Inodes != 0 {
dAtA[i] = 0x30
i++
i = encodeVarintRecord(dAtA, i, uint64(m.Inodes))
}
if m.Size_ != 0 {
dAtA[i] = 0x38
i++
i = encodeVarintRecord(dAtA, i, uint64(m.Size_))
}
return i, nil
}
func encodeFixed64Record(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Record(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintRecord(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *Snapshot) Size() (n int) {
var l int
_ = l
if m.ID != 0 {
n += 1 + sovRecord(uint64(m.ID))
}
l = len(m.Parent)
if l > 0 {
n += 1 + l + sovRecord(uint64(l))
}
if m.Kind != 0 {
n += 1 + sovRecord(uint64(m.Kind))
}
if m.Inodes != 0 {
n += 1 + sovRecord(uint64(m.Inodes))
}
if m.Size_ != 0 {
n += 1 + sovRecord(uint64(m.Size_))
}
return n
}
func sovRecord(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozRecord(x uint64) (n int) {
return sovRecord(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (this *Snapshot) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&Snapshot{`,
`ID:` + fmt.Sprintf("%v", this.ID) + `,`,
`Parent:` + fmt.Sprintf("%v", this.Parent) + `,`,
`Kind:` + fmt.Sprintf("%v", this.Kind) + `,`,
`Inodes:` + fmt.Sprintf("%v", this.Inodes) + `,`,
`Size_:` + fmt.Sprintf("%v", this.Size_) + `,`,
`}`,
}, "")
return s
}
func valueToStringRecord(v interface{}) string {
rv := reflect.ValueOf(v)
if rv.IsNil() {
return "nil"
}
pv := reflect.Indirect(rv).Interface()
return fmt.Sprintf("*%v", pv)
}
func (m *Snapshot) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Snapshot: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Snapshot: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
}
m.ID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.ID |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Parent", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthRecord
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Parent = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType)
}
m.Kind = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Kind |= (Kind(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Inodes", wireType)
}
m.Inodes = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Inodes |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType)
}
m.Size_ = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Size_ |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipRecord(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthRecord
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipRecord(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthRecord
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipRecord(dAtA[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthRecord = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowRecord = fmt.Errorf("proto: integer overflow")
)
func init() {
proto1.RegisterFile("github.com/containerd/containerd/snapshot/storage/proto/record.proto", fileDescriptorRecord)
}
var fileDescriptorRecord = []byte{
// 364 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xbb, 0xae, 0xd3, 0x30,
0x18, 0xc7, 0xe3, 0x9c, 0x28, 0xe7, 0xd4, 0x94, 0x12, 0x2c, 0x14, 0x45, 0x51, 0x65, 0x2c, 0xa6,
0x88, 0x21, 0xe1, 0xf2, 0x04, 0xbd, 0x0d, 0x51, 0xd5, 0x56, 0x0a, 0xbd, 0xcc, 0x69, 0x6c, 0xa5,
0x56, 0x55, 0xbb, 0x4a, 0x4c, 0x2b, 0x31, 0x31, 0x56, 0x79, 0x02, 0x96, 0x4c, 0xf0, 0x14, 0x3c,
0x41, 0x47, 0x46, 0x26, 0x44, 0xf3, 0x24, 0x28, 0x69, 0x11, 0x0c, 0x67, 0xfb, 0x5f, 0x7e, 0xf6,
0xf7, 0xe9, 0x83, 0xc3, 0x94, 0xab, 0xcd, 0xc7, 0xb5, 0x9f, 0xc8, 0x5d, 0x90, 0x48, 0xa1, 0x62,
0x2e, 0x58, 0x46, 0xff, 0x97, 0xb9, 0x88, 0xf7, 0xf9, 0x46, 0xaa, 0x20, 0x57, 0x32, 0x8b, 0x53,
0x16, 0xec, 0x33, 0xa9, 0x64, 0x90, 0xb1, 0x44, 0x66, 0xd4, 0x6f, 0x0c, 0xb2, 0xff, 0xf1, 0xfe,
0x5f, 0xde, 0x3f, 0xbc, 0x75, 0x5f, 0xa4, 0x32, 0x95, 0x57, 0xbe, 0x56, 0x57, 0xfa, 0xd5, 0x17,
0x00, 0x1f, 0x3e, 0xdc, 0x28, 0x64, 0x43, 0x9d, 0x53, 0x07, 0x10, 0xe0, 0x19, 0x7d, 0xb3, 0xfa,
0xf5, 0x52, 0x0f, 0x87, 0x91, 0xce, 0x29, 0xb2, 0xa1, 0xb9, 0x8f, 0x33, 0x26, 0x94, 0xa3, 0x13,
0xe0, 0xb5, 0xa2, 0x9b, 0x43, 0x6f, 0xa0, 0xb1, 0xe5, 0x82, 0x3a, 0x06, 0x01, 0x5e, 0xe7, 0x5d,
0xd7, 0x7f, 0x7c, 0xb2, 0x3f, 0xe6, 0x82, 0x46, 0x0d, 0x59, 0xff, 0xc4, 0x85, 0xa4, 0x2c, 0x77,
0x4c, 0x02, 0xbc, 0xbb, 0xe8, 0xe6, 0x10, 0x82, 0x46, 0xce, 0x3f, 0x31, 0xe7, 0xbe, 0x49, 0x1b,
0xfd, 0xfa, 0x04, 0xa0, 0x51, 0x3f, 0x45, 0x5d, 0x78, 0xbf, 0x98, 0x8e, 0xa7, 0xb3, 0xd5, 0xd4,
0xd2, 0xdc, 0x67, 0x45, 0x49, 0x9e, 0xd4, 0xf1, 0x42, 0x6c, 0x85, 0x3c, 0x0a, 0x64, 0x43, 0x63,
0x19, 0x8e, 0x56, 0x16, 0x70, 0xdb, 0x45, 0x49, 0x1e, 0xea, 0x6a, 0xc9, 0xd9, 0x11, 0xb9, 0xd0,
0xec, 0x0d, 0xe6, 0xe1, 0x72, 0x64, 0xe9, 0x6e, 0xa7, 0x28, 0x09, 0xac, 0x9b, 0x5e, 0xa2, 0xf8,
0x81, 0x21, 0x02, 0x5b, 0x83, 0xd9, 0x64, 0x12, 0xce, 0xe7, 0xa3, 0xa1, 0x75, 0xe7, 0x3e, 0x2f,
0x4a, 0xf2, 0xb4, 0xae, 0x07, 0x72, 0xb7, 0xe3, 0x4a, 0x31, 0xea, 0xb6, 0x4f, 0x5f, 0xb1, 0xf6,
0xfd, 0x1b, 0x6e, 0x36, 0xe8, 0x3b, 0xe7, 0x0b, 0xd6, 0x7e, 0x5e, 0xb0, 0xf6, 0xb9, 0xc2, 0xe0,
0x5c, 0x61, 0xf0, 0xa3, 0xc2, 0xe0, 0x77, 0x85, 0xc1, 0xda, 0x6c, 0xce, 0xf8, 0xfe, 0x4f, 0x00,
0x00, 0x00, 0xff, 0xff, 0x61, 0xef, 0x92, 0x3d, 0xbc, 0x01, 0x00, 0x00,
}

View File

@ -1,34 +0,0 @@
syntax = "proto3";
package containerd.snapshot.v1;
import "gogoproto/gogo.proto";
enum Kind {
option (gogoproto.goproto_enum_prefix) = false;
option (gogoproto.enum_customname) = "Kind";
UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "KindUnknown"];
VIEW = 1 [(gogoproto.enumvalue_customname) = "KindView"];
ACTIVE = 2 [(gogoproto.enumvalue_customname) = "KindActive"];
COMMITTED = 3 [(gogoproto.enumvalue_customname) = "KindCommitted"];
}
// Snapshot defines the storage type for a snapshot in the
// metadata store.
message Snapshot {
uint64 id = 1;
string parent = 2;
Kind kind = 4;
// inodes stores the number inodes in use for the snapshot.
//
// Only valid for committed snapshots.
int64 inodes = 6;
// Size reports the disk used by the snapshot, excluding the parents.
//
// Only valid for committed snapshots, active snapshots must read the
// current usage from the disk.
int64 size = 7;
}

View File

@ -7,7 +7,9 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
"time"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/fs/fstest" "github.com/containerd/containerd/fs/fstest"
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/namespaces"
@ -23,6 +25,8 @@ func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.
t.Run("StatComitted", makeTest(t, name, snapshotterFn, checkSnapshotterStatCommitted)) t.Run("StatComitted", makeTest(t, name, snapshotterFn, checkSnapshotterStatCommitted))
t.Run("TransitivityTest", makeTest(t, name, snapshotterFn, checkSnapshotterTransitivity)) t.Run("TransitivityTest", makeTest(t, name, snapshotterFn, checkSnapshotterTransitivity))
t.Run("PreareViewFailingtest", makeTest(t, name, snapshotterFn, checkSnapshotterPrepareView)) t.Run("PreareViewFailingtest", makeTest(t, name, snapshotterFn, checkSnapshotterPrepareView))
t.Run("Update", makeTest(t, name, snapshotterFn, checkUpdate))
t.Run("Remove", makeTest(t, name, snapshotterFn, checkRemove))
t.Run("LayerFileupdate", makeTest(t, name, snapshotterFn, checkLayerFileUpdate)) t.Run("LayerFileupdate", makeTest(t, name, snapshotterFn, checkLayerFileUpdate))
t.Run("RemoveDirectoryInLowerLayer", makeTest(t, name, snapshotterFn, checkRemoveDirectoryInLowerLayer)) t.Run("RemoveDirectoryInLowerLayer", makeTest(t, name, snapshotterFn, checkRemoveDirectoryInLowerLayer))
@ -414,3 +418,220 @@ func checkSnapshotterPrepareView(ctx context.Context, t *testing.T, snapshotter
assert.NotNil(t, err) assert.NotNil(t, err)
} }
// baseTestSnapshots creates a base set of snapshots for tests, each snapshot is empty
// Tests snapshots:
// c1 - committed snapshot, no parent
// c2 - commited snapshot, c1 is parent
// a1 - active snapshot, c2 is parent
// a1 - active snapshot, no parent
// v1 - view snapshot, v1 is parent
// v2 - view snapshot, no parent
func baseTestSnapshots(ctx context.Context, snapshotter snapshot.Snapshotter) error {
if _, err := snapshotter.Prepare(ctx, "c1-a", ""); err != nil {
return err
}
if err := snapshotter.Commit(ctx, "c1", "c1-a"); err != nil {
return err
}
if _, err := snapshotter.Prepare(ctx, "c2-a", "c1"); err != nil {
return err
}
if err := snapshotter.Commit(ctx, "c2", "c2-a"); err != nil {
return err
}
if _, err := snapshotter.Prepare(ctx, "a1", "c2"); err != nil {
return err
}
if _, err := snapshotter.Prepare(ctx, "a2", ""); err != nil {
return err
}
if _, err := snapshotter.View(ctx, "v1", "c2"); err != nil {
return err
}
if _, err := snapshotter.View(ctx, "v2", ""); err != nil {
return err
}
return nil
}
func checkUpdate(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
t1 := time.Now().UTC()
if err := baseTestSnapshots(ctx, snapshotter); err != nil {
t.Fatalf("Failed to create base snapshots: %v", err)
}
t2 := time.Now().UTC()
testcases := []struct {
name string
kind snapshot.Kind
parent string
}{
{
name: "c1",
kind: snapshot.KindCommitted,
},
{
name: "c2",
kind: snapshot.KindCommitted,
parent: "c1",
},
{
name: "a1",
kind: snapshot.KindActive,
parent: "c2",
},
{
name: "a2",
kind: snapshot.KindActive,
},
{
name: "v1",
kind: snapshot.KindView,
parent: "c2",
},
{
name: "v2",
kind: snapshot.KindView,
},
}
for _, tc := range testcases {
st, err := snapshotter.Stat(ctx, tc.name)
if err != nil {
t.Fatalf("Failed to stat %s: %v", tc.name, err)
}
if st.Created.Before(t1) || st.Created.After(t2) {
t.Errorf("(%s) wrong created time %s: expected between %s and %s", tc.name, st.Created, t1, t2)
continue
}
if st.Created != st.Updated {
t.Errorf("(%s) unexpected updated time %s: expected %s", tc.name, st.Updated, st.Created)
continue
}
if st.Kind != tc.kind {
t.Errorf("(%s) unexpected kind %s, expected %s", tc.name, st.Kind, tc.kind)
continue
}
if st.Parent != tc.parent {
t.Errorf("(%s) unexpected parent %q, expected %q", tc.name, st.Parent, tc.parent)
continue
}
if st.Name != tc.name {
t.Errorf("(%s) unexpected name %q, expected %q", tc.name, st.Name, tc.name)
continue
}
createdAt := st.Created
expected := map[string]string{
"l1": "v1",
"l2": "v2",
"l3": "v3",
}
st.Parent = "doesnotexist"
st.Labels = expected
u1 := time.Now().UTC()
st, err = snapshotter.Update(ctx, st)
if err != nil {
t.Fatalf("Failed to update %s: %v", tc.name, err)
}
u2 := time.Now().UTC()
if st.Created != createdAt {
t.Errorf("(%s) wrong created time %s: expected %s", tc.name, st.Created, createdAt)
continue
}
if st.Updated.Before(u1) || st.Updated.After(u2) {
t.Errorf("(%s) wrong updated time %s: expected between %s and %s", tc.name, st.Updated, u1, u2)
continue
}
if st.Kind != tc.kind {
t.Errorf("(%s) unexpected kind %s, expected %s", tc.name, st.Kind, tc.kind)
continue
}
if st.Parent != tc.parent {
t.Errorf("(%s) unexpected parent %q, expected %q", tc.name, st.Parent, tc.parent)
continue
}
if st.Name != tc.name {
t.Errorf("(%s) unexpected name %q, expected %q", tc.name, st.Name, tc.name)
continue
}
assertLabels(t, st.Labels, expected)
expected = map[string]string{
"l1": "updated",
"l3": "v3",
}
st.Labels = map[string]string{
"l1": "updated",
"l4": "v4",
}
st, err = snapshotter.Update(ctx, st, "labels.l1", "labels.l2")
if err != nil {
t.Fatalf("Failed to update %s: %v", tc.name, err)
}
assertLabels(t, st.Labels, expected)
expected = map[string]string{
"l4": "v4",
}
st.Labels = expected
st, err = snapshotter.Update(ctx, st, "labels")
if err != nil {
t.Fatalf("Failed to update %s: %v", tc.name, err)
}
assertLabels(t, st.Labels, expected)
// Test failure received when providing immutable field path
st.Parent = "doesnotexist"
st, err = snapshotter.Update(ctx, st, "parent")
if err == nil {
t.Errorf("Expected error updating with immutable field path")
} else if !errdefs.IsInvalidArgument(err) {
t.Fatalf("Unexpected error updating %s: %+v", tc.name, err)
}
}
}
func assertLabels(t *testing.T, actual, expected map[string]string) {
if len(actual) != len(expected) {
t.Fatalf("Label size mismatch: %d vs %d\n\tActual: %#v\n\tExpected: %#v", len(actual), len(expected), actual, expected)
}
for k, v := range expected {
if a := actual[k]; v != a {
t.Errorf("Wrong label value for %s, got %q, expected %q", k, a, v)
}
}
if t.Failed() {
t.FailNow()
}
}
func checkRemove(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
if _, err := snapshotter.Prepare(ctx, "committed-a", ""); err != nil {
t.Fatal(err)
}
if err := snapshotter.Commit(ctx, "committed-1", "committed-a"); err != nil {
t.Fatal(err)
}
if _, err := snapshotter.Prepare(ctx, "reuse-1", "committed-1"); err != nil {
t.Fatal(err)
}
if err := snapshotter.Remove(ctx, "reuse-1"); err != nil {
t.Fatal(err)
}
if _, err := snapshotter.View(ctx, "reuse-1", "committed-1"); err != nil {
t.Fatal(err)
}
if err := snapshotter.Remove(ctx, "reuse-1"); err != nil {
t.Fatal(err)
}
if _, err := snapshotter.Prepare(ctx, "reuse-1", ""); err != nil {
t.Fatal(err)
}
if err := snapshotter.Remove(ctx, "committed-1"); err != nil {
t.Fatal(err)
}
if err := snapshotter.Commit(ctx, "commited-1", "reuse-1"); err != nil {
t.Fatal(err)
}
}

View File

@ -44,15 +44,19 @@ func (o *Snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, erro
panic("not implemented") panic("not implemented")
} }
func (o *Snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) {
panic("not implemented")
}
func (o *Snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { func (o *Snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) {
panic("not implemented") panic("not implemented")
} }
func (o *Snapshotter) Prepare(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (o *Snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
panic("not implemented") panic("not implemented")
} }
func (o *Snapshotter) View(ctx context.Context, key, parent string) ([]mount.Mount, error) { func (o *Snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) {
panic("not implemented") panic("not implemented")
} }
@ -64,7 +68,7 @@ func (o *Snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
panic("not implemented") panic("not implemented")
} }
func (o *Snapshotter) Commit(ctx context.Context, name, key string) error { func (o *Snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error {
panic("not implemented") panic("not implemented")
} }