github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

snapshot: add Usage method to Snapshotter

Browse Source 
To allow the querying of usage for snapshots, we define a new method on
the snapshotter to query the resources in use by a single snapshot.
Conversely, it can be said that if the snapshot was deleted, the
reported amount of usage would be recovered.

There are few problems with this model in the implementation of btrfs
that need to be worked out. In btrfs, it is hard to resolve the amount
of data usage with the use of quotas but these may report valuables that
are incompatible with the model.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day
2017-04-12 17:56:04 -07:00
parent 4a749ad473
commit 66c504d1bb
13 changed files with 367 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
snapshot/storage/bolt.go
View File

@@ -63,21 +63,26 @@ func getParentPrefix(b []byte) uint64 {
// GetInfo returns the snapshot Info directly from the metadata. Requires a
// context with a storage transaction.
func GetInfo(ctx context.Context, key string) (snapshot.Info, error) {
func GetInfo(ctx context.Context, key string) (string, snapshot.Info, snapshot.Usage, error) {
var ss db.Snapshot
err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
return getSnapshot(bkt, key, &ss)
})
if err != nil {
return snapshot.Info{}, err
return "", snapshot.Info{}, snapshot.Usage{}, err
}
return snapshot.Info{
usage := snapshot.Usage{
Inodes: ss.Inodes,
Size: ss.Size_,
}
return fmt.Sprint(ss.ID), snapshot.Info{
Name: key,
Parent: ss.Parent,
Kind: fromProtoKind(ss.Kind),
Readonly: ss.Readonly,
}, nil
}, usage, nil
}
// WalkInfo iterates through all metadata Info for the stored snapshots and
@@ -263,7 +268,7 @@ func Remove(ctx context.Context, key string) (id string, k snapshot.Kind, err er
// lookup or removal. The returned string identifier for the committed snapshot
// is the same identifier of the original active snapshot. The provided context
// must contain a writable transaction.
func CommitActive(ctx context.Context, key, name string) (id string, err error) {
func CommitActive(ctx context.Context, key, name string, usage snapshot.Usage) (id string, err error) {
err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
b := bkt.Get([]byte(name))
if len(b) != 0 {
@@ -283,6 +288,8 @@ func CommitActive(ctx context.Context, key, name string) (id string, err error)
ss.Kind = db.KindCommitted
ss.Readonly = true
ss.Inodes = usage.Inodes
ss.Size_ = usage.Size
if err := putSnapshot(bkt, name, &ss); err != nil {
return err

14
snapshot/storage/metastore_bench_test.go
View File

@@ -6,6 +6,8 @@ import (
"io/ioutil"
"os"
"testing"
"github.com/containerd/containerd/snapshot"
)
// Benchmarks returns a benchmark suite using the provided metadata store
@@ -107,7 +109,7 @@ func createActiveFromBase(ctx context.Context, ms *MetaStore, active, base strin
if _, err := CreateActive(ctx, "bottom", "", false); err != nil {
return err
}
if _, err := CommitActive(ctx, "bottom", base); err != nil {
if _, err := CommitActive(ctx, "bottom", base, snapshot.Usage{}); err != nil {
return err
}
@@ -122,7 +124,7 @@ func statActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := GetInfo(ctx, "active")
_, _, _, err := GetInfo(ctx, "active")
if err != nil {
b.Fatal(err)
}
@@ -133,13 +135,13 @@ func statCommittedBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil {
b.Fatal(err)
}
if _, err := CommitActive(ctx, "active", "committed"); err != nil {
if _, err := CommitActive(ctx, "active", "committed", snapshot.Usage{}); err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := GetInfo(ctx, "committed")
_, _, _, err := GetInfo(ctx, "committed")
if err != nil {
b.Fatal(err)
}
@@ -179,7 +181,7 @@ func commitBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
b.Fatal(err)
}
b.StartTimer()
if _, err := CommitActive(ctx, "active", "committed"); err != nil {
if _, err := CommitActive(ctx, "active", "committed", snapshot.Usage{}); err != nil {
b.Fatal(err)
}
b.StopTimer()
@@ -196,7 +198,7 @@ func getActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
b.Fatalf("create active failed: %+v", err)
}
base = fmt.Sprintf("base-%d", i)
if _, err := CommitActive(ctx, "tmp", base); err != nil {
if _, err := CommitActive(ctx, "tmp", base, snapshot.Usage{}); err != nil {
b.Fatalf("commit failed: %+v", err)
}

24
snapshot/storage/metastore_test.go
View File

@@ -128,13 +128,13 @@ func basePopulate(ctx context.Context, ms *MetaStore) error {
if _, err := CreateActive(ctx, "committed-tmp-1", "", false); err != nil {
return errors.Wrap(err, "failed to create active")
}
if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1"); err != nil {
if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1", snapshot.Usage{Size: 1}); err != nil {
return errors.Wrap(err, "failed to create active")
}
if _, err := CreateActive(ctx, "committed-tmp-2", "committed-1", false); err != nil {
return errors.Wrap(err, "failed to create active")
}
if _, err := CommitActive(ctx, "committed-tmp-2", "committed-2"); err != nil {
if _, err := CommitActive(ctx, "committed-tmp-2", "committed-2", snapshot.Usage{Size: 2}); err != nil {
return errors.Wrap(err, "failed to create active")
}
if _, err := CreateActive(ctx, "active-1", "", false); err != nil {
@@ -238,7 +238,7 @@ func assertExist(t *testing.T, err error) {
func testGetInfo(ctx context.Context, t *testing.T, ms *MetaStore) {
for key, expected := range baseInfo {
info, err := GetInfo(ctx, key)
_, info, _, err := GetInfo(ctx, key)
if err != nil {
t.Fatalf("GetInfo on %v failed: %+v", key, err)
}
@@ -247,7 +247,7 @@ func testGetInfo(ctx context.Context, t *testing.T, ms *MetaStore) {
}
func testGetInfoNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, err := GetInfo(ctx, "active-not-exist")
_, _, _, err := GetInfo(ctx, "active-not-exist")
assertNotExist(t, err)
}
@@ -272,7 +272,7 @@ func testGetActive(ctx context.Context, t *testing.T, ms *MetaStore) {
if _, err := CreateActive(ctx, "committed-tmp-1", "", false); err != nil {
return errors.Wrap(err, "failed to create active")
}
if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1"); err != nil {
if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1", snapshot.Usage{}); err != nil {
return errors.Wrap(err, "failed to create active")
}
@@ -350,7 +350,7 @@ func testCreateActive(ctx context.Context, t *testing.T, ms *MetaStore) {
t.Fatal("Expected readonly active")
}
commitID, err := CommitActive(ctx, "active-1", "committed-1")
commitID, err := CommitActive(ctx, "active-1", "committed-1", snapshot.Usage{})
if err != nil {
t.Fatal(err)
}
@@ -425,7 +425,7 @@ func testCommit(ctx context.Context, t *testing.T, ms *MetaStore) {
t.Fatal("Expected writable active")
}
commitID, err := CommitActive(ctx, "active-1", "committed-1")
commitID, err := CommitActive(ctx, "active-1", "committed-1", snapshot.Usage{})
if err != nil {
t.Fatal(err)
}
@@ -440,7 +440,7 @@ func testCommit(ctx context.Context, t *testing.T, ms *MetaStore) {
}
func testCommitNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, err := CommitActive(ctx, "active-not-exist", "committed-1")
_, err := CommitActive(ctx, "active-not-exist", "committed-1", snapshot.Usage{})
assertNotExist(t, err)
}
@@ -448,7 +448,7 @@ func testCommitExist(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err)
}
_, err := CommitActive(ctx, "active-1", "committed-1")
_, err := CommitActive(ctx, "active-1", "committed-1", snapshot.Usage{})
assertExist(t, err)
}
@@ -456,7 +456,7 @@ func testCommitCommitted(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err)
}
_, err := CommitActive(ctx, "committed-1", "committed-3")
_, err := CommitActive(ctx, "committed-1", "committed-3", snapshot.Usage{})
assertNotActive(t, err)
}
@@ -464,7 +464,7 @@ func testCommitReadonly(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err)
}
_, err := CommitActive(ctx, "active-5", "committed-3")
_, err := CommitActive(ctx, "active-5", "committed-3", snapshot.Usage{})
if err == nil {
t.Fatal("Expected error committing readonly active")
}
@@ -476,7 +476,7 @@ func testRemove(ctx context.Context, t *testing.T, ms *MetaStore) {
t.Fatal(err)
}
commitID, err := CommitActive(ctx, "active-1", "committed-1")
commitID, err := CommitActive(ctx, "active-1", "committed-1", snapshot.Usage{})
if err != nil {
t.Fatal(err)
}

107
snapshot/storage/proto/record.pb.go
View File

@@ -65,6 +65,15 @@ type Snapshot struct {
Parent string `protobuf:"bytes,2,opt,name=parent,proto3" json:"parent,omitempty"`
Kind Kind `protobuf:"varint,4,opt,name=kind,proto3,enum=containerd.v1.Kind" json:"kind,omitempty"`
Readonly bool `protobuf:"varint,5,opt,name=readonly,proto3" json:"readonly,omitempty"`
// inodes stores the number inodes in use for the snapshot.
//
// Only valid for committed snapshots.
Inodes int64 `protobuf:"varint,6,opt,name=inodes,proto3" json:"inodes,omitempty"`
// Size reports the disk used by the snapshot, excluding the parents.
//
// Only valid for committed snapshots, active snapshots must read the
// current usage from the disk.
Size_ int64 `protobuf:"varint,7,opt,name=size,proto3" json:"size,omitempty"`
}
func (m *Snapshot) Reset() { *m = Snapshot{} }
@@ -116,6 +125,16 @@ func (m *Snapshot) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.Inodes != 0 {
dAtA[i] = 0x30
i++
i = encodeVarintRecord(dAtA, i, uint64(m.Inodes))
}
if m.Size_ != 0 {
dAtA[i] = 0x38
i++
i = encodeVarintRecord(dAtA, i, uint64(m.Size_))
}
return i, nil
}
@@ -162,6 +181,12 @@ func (m *Snapshot) Size() (n int) {
if m.Readonly {
n += 2
}
if m.Inodes != 0 {
n += 1 + sovRecord(uint64(m.Inodes))
}
if m.Size_ != 0 {
n += 1 + sovRecord(uint64(m.Size_))
}
return n
}
@@ -187,6 +212,8 @@ func (this *Snapshot) String() string {
`Parent:` + fmt.Sprintf("%v", this.Parent) + `,`,
`Kind:` + fmt.Sprintf("%v", this.Kind) + `,`,
`Readonly:` + fmt.Sprintf("%v", this.Readonly) + `,`,
`Inodes:` + fmt.Sprintf("%v", this.Inodes) + `,`,
`Size_:` + fmt.Sprintf("%v", this.Size_) + `,`,
`}`,
}, "")
return s
@@ -315,6 +342,44 @@ func (m *Snapshot) Unmarshal(dAtA []byte) error {
}
}
m.Readonly = bool(v != 0)
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Inodes", wireType)
}
m.Inodes = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Inodes |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType)
}
m.Size_ = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Size_ |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipRecord(dAtA[iNdEx:])
@@ -446,24 +511,26 @@ func init() {
}
var fileDescriptorRecord = []byte{
// 304 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x72, 0x49, 0xcf, 0x2c, 0xc9,
0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xce, 0xcf, 0x2b, 0x49, 0xcc, 0xcc, 0x4b, 0x2d,
0x4a, 0x41, 0x66, 0x16, 0xe7, 0x25, 0x16, 0x14, 0x67, 0xe4, 0x97, 0xe8, 0x17, 0x97, 0xe4, 0x17,
0x25, 0xa6, 0xa7, 0xea, 0x17, 0x14, 0xe5, 0x97, 0xe4, 0xeb, 0x17, 0xa5, 0x26, 0xe7, 0x17, 0xa5,
0xe8, 0x81, 0x39, 0x42, 0xbc, 0x08, 0xf5, 0x7a, 0x65, 0x86, 0x52, 0x22, 0xe9, 0xf9, 0xe9, 0xf9,
0x10, 0x65, 0x20, 0x16, 0x44, 0x91, 0x52, 0x3d, 0x17, 0x47, 0x30, 0xd4, 0x2c, 0x21, 0x31, 0x2e,
0xa6, 0xcc, 0x14, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x16, 0x27, 0xb6, 0x47, 0xf7, 0xe4, 0x99, 0x3c,
0x5d, 0x82, 0x98, 0x32, 0x53, 0x84, 0xc4, 0xb8, 0xd8, 0x0a, 0x12, 0x8b, 0x52, 0xf3, 0x4a, 0x24,
0x98, 0x14, 0x18, 0x35, 0x38, 0x83, 0xa0, 0x3c, 0x21, 0x75, 0x2e, 0x96, 0xec, 0xcc, 0xbc, 0x14,
0x09, 0x16, 0x05, 0x46, 0x0d, 0x3e, 0x23, 0x61, 0x3d, 0x14, 0xfb, 0xf4, 0xbc, 0x33, 0xf3, 0x52,
0x82, 0xc0, 0x0a, 0x84, 0xa4, 0xb8, 0x38, 0x8a, 0x52, 0x13, 0x53, 0xf2, 0xf3, 0x72, 0x2a, 0x25,
0x58, 0x15, 0x18, 0x35, 0x38, 0x82, 0xe0, 0x7c, 0xad, 0x20, 0x2e, 0x16, 0x6f, 0x88, 0x1a, 0x36,
0x47, 0xe7, 0x10, 0xcf, 0x30, 0x57, 0x01, 0x06, 0x29, 0xbe, 0xae, 0xb9, 0x0a, 0x5c, 0x20, 0x51,
0xc7, 0xe4, 0x92, 0xcc, 0xb2, 0x54, 0x21, 0x05, 0x2e, 0x4e, 0x67, 0x7f, 0x5f, 0x5f, 0xcf, 0x90,
0x10, 0x57, 0x17, 0x01, 0x46, 0x29, 0xc1, 0xae, 0xb9, 0x0a, 0xbc, 0x20, 0x69, 0xe7, 0xfc, 0xdc,
0xdc, 0xcc, 0x92, 0x92, 0xd4, 0x14, 0x29, 0x9e, 0x8e, 0xc5, 0x72, 0x0c, 0xbb, 0x96, 0xc8, 0x81,
0xcd, 0x72, 0x92, 0x38, 0xf1, 0x50, 0x8e, 0xe1, 0xc6, 0x43, 0x39, 0x86, 0x86, 0x47, 0x72, 0x8c,
0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91, 0x1c, 0x63, 0x12, 0x1b, 0xd8,
0xd7, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x89, 0x2c, 0x37, 0x62, 0x01, 0x00, 0x00,
// 330 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x90, 0xbd, 0x4e, 0xf3, 0x30,
0x18, 0x85, 0xe3, 0x7c, 0xf9, 0x42, 0x6b, 0xd1, 0xaa, 0x18, 0x54, 0x59, 0x19, 0x8c, 0xc5, 0x42,
0xc4, 0x90, 0x08, 0xb8, 0x82, 0xfe, 0x0d, 0x55, 0x55, 0x21, 0x85, 0x8a, 0x3d, 0x8d, 0xad, 0xd4,
0x82, 0xda, 0x95, 0x63, 0x2a, 0xc1, 0xc4, 0x88, 0x7a, 0x0f, 0x9d, 0x60, 0xe3, 0x0e, 0xb8, 0x82,
0x8e, 0x8c, 0x4c, 0x88, 0xe6, 0x4a, 0x50, 0xd2, 0x8a, 0x9f, 0xed, 0x9c, 0xe3, 0x47, 0x8f, 0xa5,
0x17, 0x76, 0x53, 0x61, 0x26, 0xb7, 0xe3, 0x20, 0x51, 0xd3, 0x30, 0x51, 0xd2, 0xc4, 0x42, 0x72,
0xcd, 0x7e, 0xc7, 0x4c, 0xc6, 0xb3, 0x6c, 0xa2, 0x4c, 0x98, 0x19, 0xa5, 0xe3, 0x94, 0x87, 0x33,
0xad, 0x8c, 0x0a, 0x35, 0x4f, 0x94, 0x66, 0x41, 0x59, 0x50, 0xed, 0x87, 0x0f, 0xe6, 0xa7, 0xde,
0x41, 0xaa, 0x52, 0xb5, 0xc1, 0x8a, 0xb4, 0x81, 0x8e, 0x5e, 0x00, 0xac, 0x5c, 0x6e, 0x65, 0xa8,
0x09, 0x6d, 0xc1, 0x30, 0xa0, 0xc0, 0x77, 0xda, 0x6e, 0xfe, 0x71, 0x68, 0xf7, 0xbb, 0x91, 0x2d,
0x18, 0x6a, 0x42, 0x77, 0x16, 0x6b, 0x2e, 0x0d, 0xb6, 0x29, 0xf0, 0xab, 0xd1, 0xb6, 0xa1, 0x63,
0xe8, 0x5c, 0x0b, 0xc9, 0xb0, 0x43, 0x81, 0x5f, 0x3f, 0xdb, 0x0f, 0xfe, 0x7c, 0x18, 0x0c, 0x84,
0x64, 0x51, 0x09, 0x20, 0x0f, 0x56, 0x34, 0x8f, 0x99, 0x92, 0x37, 0x77, 0xf8, 0x3f, 0x05, 0x7e,
0x25, 0xfa, 0xee, 0x85, 0x5c, 0x48, 0xc5, 0x78, 0x86, 0x5d, 0x0a, 0xfc, 0x7f, 0xd1, 0xb6, 0x21,
0x04, 0x9d, 0x4c, 0xdc, 0x73, 0xbc, 0x53, 0xae, 0x65, 0x3e, 0x89, 0xa0, 0x33, 0xd8, 0xf8, 0xdc,
0x56, 0x67, 0xd4, 0xbf, 0xea, 0x35, 0x2c, 0xaf, 0xbe, 0x58, 0x52, 0x58, 0xac, 0xad, 0xc4, 0x88,
0x39, 0x47, 0x14, 0x56, 0x3b, 0x17, 0xc3, 0x61, 0x7f, 0x34, 0xea, 0x75, 0x1b, 0xc0, 0xdb, 0x5b,
0x2c, 0x69, 0xad, 0x78, 0xee, 0xa8, 0xe9, 0x54, 0x18, 0xc3, 0x99, 0xb7, 0xfb, 0xf8, 0x44, 0xac,
0xd7, 0x67, 0x52, 0xba, 0xda, 0x78, 0xb5, 0x26, 0xd6, 0xfb, 0x9a, 0x58, 0x0f, 0x39, 0x01, 0xab,
0x9c, 0x80, 0xb7, 0x9c, 0x80, 0xcf, 0x9c, 0x80, 0xb1, 0x5b, 0x9e, 0xe8, 0xfc, 0x2b, 0x00, 0x00,
0xff, 0xff, 0xce, 0x7e, 0x6f, 0x85, 0x8f, 0x01, 0x00, 0x00,
}

11
snapshot/storage/proto/record.proto
View File

@@ -23,4 +23,15 @@ message Snapshot {
string parent = 2;
Kind kind = 4;
bool readonly = 5;
// inodes stores the number inodes in use for the snapshot.
//
// Only valid for committed snapshots.
int64 inodes = 6;
// Size reports the disk used by the snapshot, excluding the parents.
//
// Only valid for committed snapshots, active snapshots must read the
// current usage from the disk.
int64 size = 7;
}