Move snapshots to core/snapshots
Signed-off-by: Derek McGowan <derek@mcg.dev>
This commit is contained in:
Derek McGowan
649
core/snapshots/storage/bolt.go
Normal file
649
core/snapshots/storage/bolt.go
Normal file
@@ -0,0 +1,649 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/metadata/boltutil"
|
||||
"github.com/containerd/containerd/v2/core/snapshots"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/containerd/containerd/v2/filters"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
var (
|
||||
bucketKeyStorageVersion = []byte("v1")
|
||||
bucketKeySnapshot = []byte("snapshots")
|
||||
bucketKeyParents = []byte("parents")
|
||||
|
||||
bucketKeyID = []byte("id")
|
||||
bucketKeyParent = []byte("parent")
|
||||
bucketKeyKind = []byte("kind")
|
||||
bucketKeyInodes = []byte("inodes")
|
||||
bucketKeySize = []byte("size")
|
||||
|
||||
// ErrNoTransaction is returned when an operation is attempted with
|
||||
// a context which is not inside of a transaction.
|
||||
ErrNoTransaction = errors.New("no transaction in context")
|
||||
)
|
||||
|
||||
// parentKey returns a composite key of the parent and child identifiers. The
|
||||
// parts of the key are separated by a zero byte.
|
||||
func parentKey(parent, child uint64) []byte {
|
||||
b := make([]byte, binary.Size([]uint64{parent, child})+1)
|
||||
i := binary.PutUvarint(b, parent)
|
||||
j := binary.PutUvarint(b[i+1:], child)
|
||||
return b[0 : i+j+1]
|
||||
}
|
||||
|
||||
// parentPrefixKey returns the parent part of the composite key with the
|
||||
// zero byte separator.
|
||||
func parentPrefixKey(parent uint64) []byte {
|
||||
b := make([]byte, binary.Size(parent)+1)
|
||||
i := binary.PutUvarint(b, parent)
|
||||
return b[0 : i+1]
|
||||
}
|
||||
|
||||
// getParentPrefix returns the first part of the composite key which
|
||||
// represents the parent identifier.
|
||||
func getParentPrefix(b []byte) uint64 {
|
||||
parent, _ := binary.Uvarint(b)
|
||||
return parent
|
||||
}
|
||||
|
||||
// GetInfo returns the snapshot Info directly from the metadata. Requires a
|
||||
// context with a storage transaction.
|
||||
func GetInfo(ctx context.Context, key string) (string, snapshots.Info, snapshots.Usage, error) {
|
||||
var (
|
||||
id uint64
|
||||
su snapshots.Usage
|
||||
si = snapshots.Info{
|
||||
Name: key,
|
||||
}
|
||||
)
|
||||
err := withSnapshotBucket(ctx, key, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
|
||||
getUsage(bkt, &su)
|
||||
return readSnapshot(bkt, &id, &si)
|
||||
})
|
||||
if err != nil {
|
||||
return "", snapshots.Info{}, snapshots.Usage{}, err
|
||||
}
|
||||
|
||||
return strconv.FormatUint(id, 10), si, su, nil
|
||||
}
|
||||
|
||||
// UpdateInfo updates an existing snapshot info's data
|
||||
func UpdateInfo(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) {
|
||||
updated := snapshots.Info{
|
||||
Name: info.Name,
|
||||
}
|
||||
err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
|
||||
sbkt := bkt.Bucket([]byte(info.Name))
|
||||
if sbkt == nil {
|
||||
return fmt.Errorf("snapshot does not exist: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
if err := readSnapshot(sbkt, nil, &updated); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(fieldpaths) > 0 {
|
||||
for _, path := range fieldpaths {
|
||||
if strings.HasPrefix(path, "labels.") {
|
||||
if updated.Labels == nil {
|
||||
updated.Labels = map[string]string{}
|
||||
}
|
||||
|
||||
key := strings.TrimPrefix(path, "labels.")
|
||||
updated.Labels[key] = info.Labels[key]
|
||||
continue
|
||||
}
|
||||
|
||||
switch path {
|
||||
case "labels":
|
||||
updated.Labels = info.Labels
|
||||
default:
|
||||
return fmt.Errorf("cannot update %q field on snapshot %q: %w", path, info.Name, errdefs.ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Set mutable fields
|
||||
updated.Labels = info.Labels
|
||||
}
|
||||
updated.Updated = time.Now().UTC()
|
||||
if err := boltutil.WriteTimestamps(sbkt, updated.Created, updated.Updated); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return boltutil.WriteLabels(sbkt, updated.Labels)
|
||||
})
|
||||
if err != nil {
|
||||
return snapshots.Info{}, err
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// WalkInfo iterates through all metadata Info for the stored snapshots and
|
||||
// calls the provided function for each. Requires a context with a storage
|
||||
// transaction.
|
||||
func WalkInfo(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
|
||||
filter, err := filters.ParseAll(fs...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: allow indexes (name, parent, specific labels)
|
||||
return withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
|
||||
return bkt.ForEach(func(k, v []byte) error {
|
||||
// skip non buckets
|
||||
if v != nil {
|
||||
return nil
|
||||
}
|
||||
var (
|
||||
sbkt = bkt.Bucket(k)
|
||||
si = snapshots.Info{
|
||||
Name: string(k),
|
||||
}
|
||||
)
|
||||
if err := readSnapshot(sbkt, nil, &si); err != nil {
|
||||
return err
|
||||
}
|
||||
if !filter.Match(adaptSnapshot(si)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fn(ctx, si)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// GetSnapshot returns the metadata for the active or view snapshot transaction
|
||||
// referenced by the given key. Requires a context with a storage transaction.
|
||||
func GetSnapshot(ctx context.Context, key string) (s Snapshot, err error) {
|
||||
err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
|
||||
sbkt := bkt.Bucket([]byte(key))
|
||||
if sbkt == nil {
|
||||
return fmt.Errorf("snapshot does not exist: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
s.ID = strconv.FormatUint(readID(sbkt), 10)
|
||||
s.Kind = readKind(sbkt)
|
||||
|
||||
if s.Kind != snapshots.KindActive && s.Kind != snapshots.KindView {
|
||||
return fmt.Errorf("requested snapshot %v not active or view: %w", key, errdefs.ErrFailedPrecondition)
|
||||
}
|
||||
|
||||
if parentKey := sbkt.Get(bucketKeyParent); len(parentKey) > 0 {
|
||||
spbkt := bkt.Bucket(parentKey)
|
||||
if spbkt == nil {
|
||||
return fmt.Errorf("parent does not exist: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
s.ParentIDs, err = parents(bkt, spbkt, readID(spbkt))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get parent chain: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// CreateSnapshot inserts a record for an active or view snapshot with the provided parent.
|
||||
func CreateSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts ...snapshots.Opt) (s Snapshot, err error) {
|
||||
switch kind {
|
||||
case snapshots.KindActive, snapshots.KindView:
|
||||
default:
|
||||
return Snapshot{}, fmt.Errorf("snapshot type %v invalid; only snapshots of type Active or View can be created: %w", kind, errdefs.ErrInvalidArgument)
|
||||
}
|
||||
var base snapshots.Info
|
||||
for _, opt := range opts {
|
||||
if err := opt(&base); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
}
|
||||
|
||||
err = createBucketIfNotExists(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
|
||||
var (
|
||||
spbkt *bolt.Bucket
|
||||
)
|
||||
if parent != "" {
|
||||
spbkt = bkt.Bucket([]byte(parent))
|
||||
if spbkt == nil {
|
||||
return fmt.Errorf("missing parent %q bucket: %w", parent, errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
if readKind(spbkt) != snapshots.KindCommitted {
|
||||
return fmt.Errorf("parent %q is not committed snapshot: %w", parent, errdefs.ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
sbkt, err := bkt.CreateBucket([]byte(key))
|
||||
if err != nil {
|
||||
if err == bolt.ErrBucketExists {
|
||||
err = fmt.Errorf("snapshot %v: %w", key, errdefs.ErrAlreadyExists)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
id, err := bkt.NextSequence()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to get identifier for snapshot %q: %w", key, err)
|
||||
}
|
||||
|
||||
t := time.Now().UTC()
|
||||
si := snapshots.Info{
|
||||
Parent: parent,
|
||||
Kind: kind,
|
||||
Labels: base.Labels,
|
||||
Created: t,
|
||||
Updated: t,
|
||||
}
|
||||
if err := putSnapshot(sbkt, id, si); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if spbkt != nil {
|
||||
pid := readID(spbkt)
|
||||
|
||||
// Store a backlink from the key to the parent. Store the snapshot name
|
||||
// as the value to allow following the backlink to the snapshot value.
|
||||
if err := pbkt.Put(parentKey(pid, id), []byte(key)); err != nil {
|
||||
return fmt.Errorf("failed to write parent link for snapshot %q: %w", key, err)
|
||||
}
|
||||
|
||||
s.ParentIDs, err = parents(bkt, spbkt, pid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get parent chain for snapshot %q: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
s.ID = strconv.FormatUint(id, 10)
|
||||
s.Kind = kind
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Remove removes a snapshot from the metastore. The string identifier for the
|
||||
// snapshot is returned as well as the kind. The provided context must contain a
|
||||
// writable transaction.
|
||||
func Remove(ctx context.Context, key string) (string, snapshots.Kind, error) {
|
||||
var (
|
||||
id uint64
|
||||
si snapshots.Info
|
||||
)
|
||||
|
||||
if err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
|
||||
sbkt := bkt.Bucket([]byte(key))
|
||||
if sbkt == nil {
|
||||
return fmt.Errorf("snapshot %v: %w", key, errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
if err := readSnapshot(sbkt, &id, &si); err != nil {
|
||||
return fmt.Errorf("failed to read snapshot %s: %w", key, err)
|
||||
}
|
||||
|
||||
if pbkt != nil {
|
||||
k, _ := pbkt.Cursor().Seek(parentPrefixKey(id))
|
||||
if getParentPrefix(k) == id {
|
||||
return fmt.Errorf("cannot remove snapshot with child: %w", errdefs.ErrFailedPrecondition)
|
||||
}
|
||||
|
||||
if si.Parent != "" {
|
||||
spbkt := bkt.Bucket([]byte(si.Parent))
|
||||
if spbkt == nil {
|
||||
return fmt.Errorf("snapshot %v: %w", key, errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
if err := pbkt.Delete(parentKey(readID(spbkt), id)); err != nil {
|
||||
return fmt.Errorf("failed to delete parent link: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := bkt.DeleteBucket([]byte(key)); err != nil {
|
||||
return fmt.Errorf("failed to delete snapshot: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
return strconv.FormatUint(id, 10), si.Kind, nil
|
||||
}
|
||||
|
||||
// CommitActive renames the active snapshot transaction referenced by `key`
|
||||
// as a committed snapshot referenced by `Name`. The resulting snapshot will be
|
||||
// committed and readonly. The `key` reference will no longer be available for
|
||||
// lookup or removal. The returned string identifier for the committed snapshot
|
||||
// is the same identifier of the original active snapshot. The provided context
|
||||
// must contain a writable transaction.
|
||||
func CommitActive(ctx context.Context, key, name string, usage snapshots.Usage, opts ...snapshots.Opt) (string, error) {
|
||||
var (
|
||||
id uint64
|
||||
base snapshots.Info
|
||||
)
|
||||
for _, opt := range opts {
|
||||
if err := opt(&base); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
|
||||
dbkt, err := bkt.CreateBucket([]byte(name))
|
||||
if err != nil {
|
||||
if err == bolt.ErrBucketExists {
|
||||
err = errdefs.ErrAlreadyExists
|
||||
}
|
||||
return fmt.Errorf("committed snapshot %v: %w", name, err)
|
||||
}
|
||||
sbkt := bkt.Bucket([]byte(key))
|
||||
if sbkt == nil {
|
||||
return fmt.Errorf("failed to get active snapshot %q: %w", key, errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
var si snapshots.Info
|
||||
if err := readSnapshot(sbkt, &id, &si); err != nil {
|
||||
return fmt.Errorf("failed to read active snapshot %q: %w", key, err)
|
||||
}
|
||||
|
||||
if si.Kind != snapshots.KindActive {
|
||||
return fmt.Errorf("snapshot %q is not active: %w", key, errdefs.ErrFailedPrecondition)
|
||||
}
|
||||
si.Kind = snapshots.KindCommitted
|
||||
si.Created = time.Now().UTC()
|
||||
si.Updated = si.Created
|
||||
|
||||
// Replace labels, do not inherit
|
||||
si.Labels = base.Labels
|
||||
|
||||
if err := putSnapshot(dbkt, id, si); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := putUsage(dbkt, usage); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bkt.DeleteBucket([]byte(key)); err != nil {
|
||||
return fmt.Errorf("failed to delete active snapshot %q: %w", key, err)
|
||||
}
|
||||
if si.Parent != "" {
|
||||
spbkt := bkt.Bucket([]byte(si.Parent))
|
||||
if spbkt == nil {
|
||||
return fmt.Errorf("missing parent %q of snapshot %q: %w", si.Parent, key, errdefs.ErrNotFound)
|
||||
}
|
||||
pid := readID(spbkt)
|
||||
|
||||
// Updates parent back link to use new key
|
||||
if err := pbkt.Put(parentKey(pid, id), []byte(name)); err != nil {
|
||||
return fmt.Errorf("failed to update parent link %q from %q to %q: %w", pid, key, name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return strconv.FormatUint(id, 10), nil
|
||||
}
|
||||
|
||||
// IDMap returns all the IDs mapped to their key
|
||||
func IDMap(ctx context.Context) (map[string]string, error) {
|
||||
m := map[string]string{}
|
||||
if err := withBucket(ctx, func(ctx context.Context, bkt, _ *bolt.Bucket) error {
|
||||
return bkt.ForEach(func(k, v []byte) error {
|
||||
// skip non buckets
|
||||
if v != nil {
|
||||
return nil
|
||||
}
|
||||
id := readID(bkt.Bucket(k))
|
||||
m[strconv.FormatUint(id, 10)] = string(k)
|
||||
return nil
|
||||
})
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func withSnapshotBucket(ctx context.Context, key string, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
|
||||
tx, ok := ctx.Value(transactionKey{}).(*bolt.Tx)
|
||||
if !ok {
|
||||
return ErrNoTransaction
|
||||
}
|
||||
vbkt := tx.Bucket(bucketKeyStorageVersion)
|
||||
if vbkt == nil {
|
||||
return fmt.Errorf("bucket does not exist: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
bkt := vbkt.Bucket(bucketKeySnapshot)
|
||||
if bkt == nil {
|
||||
return fmt.Errorf("snapshots bucket does not exist: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
bkt = bkt.Bucket([]byte(key))
|
||||
if bkt == nil {
|
||||
return fmt.Errorf("snapshot does not exist: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
return fn(ctx, bkt, vbkt.Bucket(bucketKeyParents))
|
||||
}
|
||||
|
||||
func withBucket(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
|
||||
tx, ok := ctx.Value(transactionKey{}).(*bolt.Tx)
|
||||
if !ok {
|
||||
return ErrNoTransaction
|
||||
}
|
||||
bkt := tx.Bucket(bucketKeyStorageVersion)
|
||||
if bkt == nil {
|
||||
return fmt.Errorf("bucket does not exist: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
return fn(ctx, bkt.Bucket(bucketKeySnapshot), bkt.Bucket(bucketKeyParents))
|
||||
}
|
||||
|
||||
func createBucketIfNotExists(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
|
||||
tx, ok := ctx.Value(transactionKey{}).(*bolt.Tx)
|
||||
if !ok {
|
||||
return ErrNoTransaction
|
||||
}
|
||||
|
||||
bkt, err := tx.CreateBucketIfNotExists(bucketKeyStorageVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create version bucket: %w", err)
|
||||
}
|
||||
sbkt, err := bkt.CreateBucketIfNotExists(bucketKeySnapshot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create snapshots bucket: %w", err)
|
||||
}
|
||||
pbkt, err := bkt.CreateBucketIfNotExists(bucketKeyParents)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create parents bucket: %w", err)
|
||||
}
|
||||
return fn(ctx, sbkt, pbkt)
|
||||
}
|
||||
|
||||
func parents(bkt, pbkt *bolt.Bucket, parent uint64) (parents []string, err error) {
|
||||
for {
|
||||
parents = append(parents, strconv.FormatUint(parent, 10))
|
||||
|
||||
parentKey := pbkt.Get(bucketKeyParent)
|
||||
if len(parentKey) == 0 {
|
||||
return
|
||||
}
|
||||
pbkt = bkt.Bucket(parentKey)
|
||||
if pbkt == nil {
|
||||
return nil, fmt.Errorf("missing parent: %w", errdefs.ErrNotFound)
|
||||
}
|
||||
|
||||
parent = readID(pbkt)
|
||||
}
|
||||
}
|
||||
|
||||
func readKind(bkt *bolt.Bucket) (k snapshots.Kind) {
|
||||
kind := bkt.Get(bucketKeyKind)
|
||||
if len(kind) == 1 {
|
||||
k = snapshots.Kind(kind[0])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func readID(bkt *bolt.Bucket) uint64 {
|
||||
id, _ := binary.Uvarint(bkt.Get(bucketKeyID))
|
||||
return id
|
||||
}
|
||||
|
||||
func readSnapshot(bkt *bolt.Bucket, id *uint64, si *snapshots.Info) error {
|
||||
if id != nil {
|
||||
*id = readID(bkt)
|
||||
}
|
||||
if si != nil {
|
||||
si.Kind = readKind(bkt)
|
||||
si.Parent = string(bkt.Get(bucketKeyParent))
|
||||
|
||||
if err := boltutil.ReadTimestamps(bkt, &si.Created, &si.Updated); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
labels, err := boltutil.ReadLabels(bkt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
si.Labels = labels
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func putSnapshot(bkt *bolt.Bucket, id uint64, si snapshots.Info) error {
|
||||
idEncoded, err := encodeID(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updates := [][2][]byte{
|
||||
{bucketKeyID, idEncoded},
|
||||
{bucketKeyKind, []byte{byte(si.Kind)}},
|
||||
}
|
||||
if si.Parent != "" {
|
||||
updates = append(updates, [2][]byte{bucketKeyParent, []byte(si.Parent)})
|
||||
}
|
||||
for _, v := range updates {
|
||||
if err := bkt.Put(v[0], v[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := boltutil.WriteTimestamps(bkt, si.Created, si.Updated); err != nil {
|
||||
return err
|
||||
}
|
||||
return boltutil.WriteLabels(bkt, si.Labels)
|
||||
}
|
||||
|
||||
func getUsage(bkt *bolt.Bucket, usage *snapshots.Usage) {
|
||||
usage.Inodes, _ = binary.Varint(bkt.Get(bucketKeyInodes))
|
||||
usage.Size, _ = binary.Varint(bkt.Get(bucketKeySize))
|
||||
}
|
||||
|
||||
func putUsage(bkt *bolt.Bucket, usage snapshots.Usage) error {
|
||||
for _, v := range []struct {
|
||||
key []byte
|
||||
value int64
|
||||
}{
|
||||
{bucketKeyInodes, usage.Inodes},
|
||||
{bucketKeySize, usage.Size},
|
||||
} {
|
||||
e, err := encodeSize(v.value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bkt.Put(v.key, e); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func encodeSize(size int64) ([]byte, error) {
|
||||
var (
|
||||
buf [binary.MaxVarintLen64]byte
|
||||
sizeEncoded = buf[:]
|
||||
)
|
||||
sizeEncoded = sizeEncoded[:binary.PutVarint(sizeEncoded, size)]
|
||||
|
||||
if len(sizeEncoded) == 0 {
|
||||
return nil, fmt.Errorf("failed encoding size = %v", size)
|
||||
}
|
||||
return sizeEncoded, nil
|
||||
}
|
||||
|
||||
func encodeID(id uint64) ([]byte, error) {
|
||||
var (
|
||||
buf [binary.MaxVarintLen64]byte
|
||||
idEncoded = buf[:]
|
||||
)
|
||||
idEncoded = idEncoded[:binary.PutUvarint(idEncoded, id)]
|
||||
|
||||
if len(idEncoded) == 0 {
|
||||
return nil, fmt.Errorf("failed encoding id = %v", id)
|
||||
}
|
||||
return idEncoded, nil
|
||||
}
|
||||
|
||||
func adaptSnapshot(info snapshots.Info) filters.Adaptor {
|
||||
return filters.AdapterFunc(func(fieldpath []string) (string, bool) {
|
||||
if len(fieldpath) == 0 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
switch fieldpath[0] {
|
||||
case "kind":
|
||||
switch info.Kind {
|
||||
case snapshots.KindActive:
|
||||
return "active", true
|
||||
case snapshots.KindView:
|
||||
return "view", true
|
||||
case snapshots.KindCommitted:
|
||||
return "committed", true
|
||||
}
|
||||
case "name":
|
||||
return info.Name, true
|
||||
case "parent":
|
||||
return info.Parent, true
|
||||
case "labels":
|
||||
if len(info.Labels) == 0 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
v, ok := info.Labels[strings.Join(fieldpath[1:], ".")]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
return "", false
|
||||
})
|
||||
}
|
||||
38
core/snapshots/storage/bolt_test.go
Normal file
38
core/snapshots/storage/bolt_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
// Does not require root but flag must be defined for snapshot tests
|
||||
|
||||
_ "github.com/containerd/containerd/v2/pkg/testutil"
|
||||
)
|
||||
|
||||
func TestMetastore(t *testing.T) {
|
||||
MetaStoreSuite(t, "Metastore", func(root string) (*MetaStore, error) {
|
||||
return NewMetaStore(filepath.Join(root, "metadata.db"))
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkSuite(b *testing.B) {
|
||||
Benchmarks(b, "BoltDBBench", func(root string) (*MetaStore, error) {
|
||||
return NewMetaStore(filepath.Join(root, "metadata.db"))
|
||||
})
|
||||
}
|
||||
157
core/snapshots/storage/metastore.go
Normal file
157
core/snapshots/storage/metastore.go
Normal file
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package storage provides a metadata storage implementation for snapshot
|
||||
// drivers. Drive implementations are responsible for starting and managing
|
||||
// transactions using the defined context creator. This storage package uses
|
||||
// BoltDB for storing metadata. Access to the raw boltdb transaction is not
|
||||
// provided, but the stored object is provided by the proto subpackage.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/snapshots"
|
||||
"github.com/containerd/log"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// Transactor is used to finalize an active transaction.
|
||||
type Transactor interface {
|
||||
// Commit commits any changes made during the transaction. On error a
|
||||
// caller is expected to clean up any resources which would have relied
|
||||
// on data mutated as part of this transaction. Only writable
|
||||
// transactions can commit, non-writable must call Rollback.
|
||||
Commit() error
|
||||
|
||||
// Rollback rolls back any changes made during the transaction. This
|
||||
// must be called on all non-writable transactions and aborted writable
|
||||
// transaction.
|
||||
Rollback() error
|
||||
}
|
||||
|
||||
// Snapshot hold the metadata for an active or view snapshot transaction. The
|
||||
// ParentIDs hold the snapshot identifiers for the committed snapshots this
|
||||
// active or view is based on. The ParentIDs are ordered from the lowest base
|
||||
// to highest, meaning they should be applied in order from the first index to
|
||||
// the last index. The last index should always be considered the active
|
||||
// snapshots immediate parent.
|
||||
type Snapshot struct {
|
||||
Kind snapshots.Kind
|
||||
ID string
|
||||
ParentIDs []string
|
||||
}
|
||||
|
||||
// MetaStore is used to store metadata related to a snapshot driver. The
|
||||
// MetaStore is intended to store metadata related to name, state and
|
||||
// parentage. Using the MetaStore is not required to implement a snapshot
|
||||
// driver but can be used to handle the persistence and transactional
|
||||
// complexities of a driver implementation.
|
||||
type MetaStore struct {
|
||||
dbfile string
|
||||
|
||||
dbL sync.Mutex
|
||||
db *bolt.DB
|
||||
}
|
||||
|
||||
// NewMetaStore returns a snapshot MetaStore for storage of metadata related to
|
||||
// a snapshot driver backed by a bolt file database. This implementation is
|
||||
// strongly consistent and does all metadata changes in a transaction to prevent
|
||||
// against process crashes causing inconsistent metadata state.
|
||||
func NewMetaStore(dbfile string) (*MetaStore, error) {
|
||||
return &MetaStore{
|
||||
dbfile: dbfile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type transactionKey struct{}
|
||||
|
||||
// TransactionContext creates a new transaction context. The writable value
|
||||
// should be set to true for transactions which are expected to mutate data.
|
||||
func (ms *MetaStore) TransactionContext(ctx context.Context, writable bool) (context.Context, Transactor, error) {
|
||||
ms.dbL.Lock()
|
||||
if ms.db == nil {
|
||||
db, err := bolt.Open(ms.dbfile, 0600, nil)
|
||||
if err != nil {
|
||||
ms.dbL.Unlock()
|
||||
return ctx, nil, fmt.Errorf("failed to open database file: %w", err)
|
||||
}
|
||||
ms.db = db
|
||||
}
|
||||
ms.dbL.Unlock()
|
||||
|
||||
tx, err := ms.db.Begin(writable)
|
||||
if err != nil {
|
||||
return ctx, nil, fmt.Errorf("failed to start transaction: %w", err)
|
||||
}
|
||||
|
||||
ctx = context.WithValue(ctx, transactionKey{}, tx)
|
||||
|
||||
return ctx, tx, nil
|
||||
}
|
||||
|
||||
// TransactionCallback represents a callback to be invoked while under a metastore transaction.
|
||||
type TransactionCallback func(ctx context.Context) error
|
||||
|
||||
// WithTransaction is a convenience method to run a function `fn` while holding a meta store transaction.
|
||||
// If the callback `fn` returns an error or the transaction is not writable, the database transaction will be discarded.
|
||||
func (ms *MetaStore) WithTransaction(ctx context.Context, writable bool, fn TransactionCallback) error {
|
||||
ctx, trans, err := ms.TransactionContext(ctx, writable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var result []error
|
||||
err = fn(ctx)
|
||||
if err != nil {
|
||||
result = append(result, err)
|
||||
}
|
||||
|
||||
// Always rollback if transaction is not writable
|
||||
if err != nil || !writable {
|
||||
if terr := trans.Rollback(); terr != nil {
|
||||
log.G(ctx).WithError(terr).Error("failed to rollback transaction")
|
||||
|
||||
result = append(result, fmt.Errorf("rollback failed: %w", terr))
|
||||
}
|
||||
} else {
|
||||
if terr := trans.Commit(); terr != nil {
|
||||
log.G(ctx).WithError(terr).Error("failed to commit transaction")
|
||||
|
||||
result = append(result, fmt.Errorf("commit failed: %w", terr))
|
||||
}
|
||||
}
|
||||
|
||||
if err := errors.Join(result...); err != nil {
|
||||
log.G(ctx).WithError(err).Debug("snapshotter error")
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the metastore and any underlying database connections
|
||||
func (ms *MetaStore) Close() error {
|
||||
ms.dbL.Lock()
|
||||
defer ms.dbL.Unlock()
|
||||
if ms.db == nil {
|
||||
return nil
|
||||
}
|
||||
return ms.db.Close()
|
||||
}
|
||||
220
core/snapshots/storage/metastore_bench_test.go
Normal file
220
core/snapshots/storage/metastore_bench_test.go
Normal file
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/snapshots"
|
||||
)
|
||||
|
||||
// Benchmarks returns a benchmark suite using the provided metadata store
|
||||
// creation method
|
||||
func Benchmarks(b *testing.B, name string, metaFn metaFactory) {
|
||||
b.Run("StatActive", makeBench(b, name, metaFn, statActiveBenchmark))
|
||||
b.Run("StatCommitted", makeBench(b, name, metaFn, statCommittedBenchmark))
|
||||
b.Run("CreateActive", makeBench(b, name, metaFn, createActiveBenchmark))
|
||||
b.Run("Remove", makeBench(b, name, metaFn, removeBenchmark))
|
||||
b.Run("Commit", makeBench(b, name, metaFn, commitBenchmark))
|
||||
b.Run("GetActive", makeBench(b, name, metaFn, getActiveBenchmark))
|
||||
b.Run("WriteTransaction", openCloseWritable(b, name, metaFn))
|
||||
b.Run("ReadTransaction", openCloseReadonly(b, name, metaFn))
|
||||
}
|
||||
|
||||
// makeBench creates a benchmark with a writable transaction
|
||||
func makeBench(b *testing.B, name string, metaFn metaFactory, fn func(context.Context, *testing.B, *MetaStore)) func(b *testing.B) {
|
||||
return func(b *testing.B) {
|
||||
ctx := context.Background()
|
||||
|
||||
ms, err := metaFn(b.TempDir())
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.Cleanup(func() {
|
||||
ms.Close()
|
||||
})
|
||||
|
||||
ctx, t, err := ms.TransactionContext(ctx, true)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer t.Commit()
|
||||
|
||||
b.ResetTimer()
|
||||
fn(ctx, b, ms)
|
||||
}
|
||||
}
|
||||
|
||||
func openCloseWritable(b *testing.B, name string, metaFn metaFactory) func(b *testing.B) {
|
||||
return func(b *testing.B) {
|
||||
ctx := context.Background()
|
||||
|
||||
ms, err := metaFn(b.TempDir())
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, t, err := ms.TransactionContext(ctx, true)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if err := t.Commit(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func openCloseReadonly(b *testing.B, name string, metaFn metaFactory) func(b *testing.B) {
|
||||
return func(b *testing.B) {
|
||||
ctx := context.Background()
|
||||
|
||||
ms, err := metaFn(b.TempDir())
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, t, err := ms.TransactionContext(ctx, false)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if err := t.Rollback(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createActiveFromBase(ctx context.Context, ms *MetaStore, active, base string) error {
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "bottom", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := CommitActive(ctx, "bottom", base, snapshots.Usage{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := CreateSnapshot(ctx, snapshots.KindActive, active, base)
|
||||
return err
|
||||
}
|
||||
|
||||
func statActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
|
||||
if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _, _, err := GetInfo(ctx, "active")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func statCommittedBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
|
||||
if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if _, err := CommitActive(ctx, "active", "committed", snapshots.Usage{}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _, _, err := GetInfo(ctx, "committed")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "active", ""); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.StopTimer()
|
||||
if _, _, err := Remove(ctx, "active"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.StartTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func removeBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StopTimer()
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "active", ""); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.StartTimer()
|
||||
if _, _, err := Remove(ctx, "active"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func commitBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
|
||||
b.StopTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "active", ""); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.StartTimer()
|
||||
if _, err := CommitActive(ctx, "active", "committed", snapshots.Usage{}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.StopTimer()
|
||||
if _, _, err := Remove(ctx, "committed"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
|
||||
var base string
|
||||
for i := 1; i <= 10; i++ {
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "tmp", base); err != nil {
|
||||
b.Fatalf("create active failed: %+v", err)
|
||||
}
|
||||
base = fmt.Sprintf("base-%d", i)
|
||||
if _, err := CommitActive(ctx, "tmp", base, snapshots.Usage{}); err != nil {
|
||||
b.Fatalf("commit failed: %+v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "active", base); err != nil {
|
||||
b.Fatalf("create active failed: %+v", err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := GetSnapshot(ctx, "active"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
642
core/snapshots/storage/metastore_test.go
Normal file
642
core/snapshots/storage/metastore_test.go
Normal file
@@ -0,0 +1,642 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/snapshots"
|
||||
"github.com/containerd/containerd/v2/errdefs"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type testFunc func(context.Context, *testing.T, *MetaStore)
|
||||
|
||||
type metaFactory func(string) (*MetaStore, error)
|
||||
|
||||
type populateFunc func(context.Context, *MetaStore) error
|
||||
|
||||
// MetaStoreSuite runs a test suite on the metastore given a factory function.
|
||||
func MetaStoreSuite(t *testing.T, name string, meta func(root string) (*MetaStore, error)) {
|
||||
t.Run("GetInfo", makeTest(t, name, meta, inReadTransaction(testGetInfo, basePopulate)))
|
||||
t.Run("GetInfoNotExist", makeTest(t, name, meta, inReadTransaction(testGetInfoNotExist, basePopulate)))
|
||||
t.Run("GetInfoEmptyDB", makeTest(t, name, meta, inReadTransaction(testGetInfoNotExist, nil)))
|
||||
t.Run("Walk", makeTest(t, name, meta, inReadTransaction(testWalk, basePopulate)))
|
||||
t.Run("GetSnapshot", makeTest(t, name, meta, testGetSnapshot))
|
||||
t.Run("GetSnapshotNotExist", makeTest(t, name, meta, inReadTransaction(testGetSnapshotNotExist, basePopulate)))
|
||||
t.Run("GetSnapshotCommitted", makeTest(t, name, meta, inReadTransaction(testGetSnapshotCommitted, basePopulate)))
|
||||
t.Run("GetSnapshotEmptyDB", makeTest(t, name, meta, inReadTransaction(testGetSnapshotNotExist, basePopulate)))
|
||||
t.Run("CreateActive", makeTest(t, name, meta, inWriteTransaction(testCreateActive)))
|
||||
t.Run("CreateActiveNotExist", makeTest(t, name, meta, inWriteTransaction(testCreateActiveNotExist)))
|
||||
t.Run("CreateActiveExist", makeTest(t, name, meta, inWriteTransaction(testCreateActiveExist)))
|
||||
t.Run("CreateActiveFromActive", makeTest(t, name, meta, inWriteTransaction(testCreateActiveFromActive)))
|
||||
t.Run("Commit", makeTest(t, name, meta, inWriteTransaction(testCommit)))
|
||||
t.Run("CommitNotExist", makeTest(t, name, meta, inWriteTransaction(testCommitExist)))
|
||||
t.Run("CommitExist", makeTest(t, name, meta, inWriteTransaction(testCommitExist)))
|
||||
t.Run("CommitCommitted", makeTest(t, name, meta, inWriteTransaction(testCommitCommitted)))
|
||||
t.Run("CommitViewFails", makeTest(t, name, meta, inWriteTransaction(testCommitViewFails)))
|
||||
t.Run("Remove", makeTest(t, name, meta, inWriteTransaction(testRemove)))
|
||||
t.Run("RemoveNotExist", makeTest(t, name, meta, inWriteTransaction(testRemoveNotExist)))
|
||||
t.Run("RemoveWithChildren", makeTest(t, name, meta, inWriteTransaction(testRemoveWithChildren)))
|
||||
t.Run("ParentIDs", makeTest(t, name, meta, inWriteTransaction(testParents)))
|
||||
}
|
||||
|
||||
// makeTest creates a testsuite with a writable transaction
|
||||
func makeTest(t *testing.T, name string, metaFn metaFactory, fn testFunc) func(t *testing.T) {
|
||||
return func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
ms, err := metaFn(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
ms.Close()
|
||||
})
|
||||
|
||||
fn(ctx, t, ms)
|
||||
}
|
||||
}
|
||||
|
||||
func inReadTransaction(fn testFunc, pf populateFunc) testFunc {
|
||||
return func(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if pf != nil {
|
||||
ctx, tx, err := ms.TransactionContext(ctx, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pf(ctx, ms); err != nil {
|
||||
if rerr := tx.Rollback(); rerr != nil {
|
||||
t.Logf("Rollback failed: %+v", rerr)
|
||||
}
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("Populate commit failed: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, tx, err := ms.TransactionContext(ctx, false)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed start transaction: %+v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := tx.Rollback(); err != nil {
|
||||
t.Logf("Rollback failed: %+v", err)
|
||||
if !t.Failed() {
|
||||
t.FailNow()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
fn(ctx, t, ms)
|
||||
}
|
||||
}
|
||||
|
||||
func inWriteTransaction(fn testFunc) testFunc {
|
||||
return func(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
ctx, tx, err := ms.TransactionContext(ctx, true)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start transaction: %+v", err)
|
||||
}
|
||||
defer func() {
|
||||
if t.Failed() {
|
||||
if err := tx.Rollback(); err != nil {
|
||||
t.Logf("Rollback failed: %+v", err)
|
||||
}
|
||||
} else {
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("Commit failed: %+v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
fn(ctx, t, ms)
|
||||
}
|
||||
}
|
||||
|
||||
// basePopulate creates 7 snapshots
|
||||
// - "committed-1": committed without parent
|
||||
// - "committed-2": committed with parent "committed-1"
|
||||
// - "active-1": active without parent
|
||||
// - "active-2": active with parent "committed-1"
|
||||
// - "active-3": active with parent "committed-2"
|
||||
// - "active-4": readonly active without parent"
|
||||
// - "active-5": readonly active with parent "committed-2"
|
||||
func basePopulate(ctx context.Context, ms *MetaStore) error {
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "committed-tmp-1", ""); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1", snapshots.Usage{Size: 1}); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "committed-tmp-2", "committed-1"); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CommitActive(ctx, "committed-tmp-2", "committed-2", snapshots.Usage{Size: 2}); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "active-1", ""); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "active-2", "committed-1"); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "active-3", "committed-2"); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindView, "view-1", ""); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindView, "view-2", "committed-2"); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var baseInfo = map[string]snapshots.Info{
|
||||
"committed-1": {
|
||||
Name: "committed-1",
|
||||
Parent: "",
|
||||
Kind: snapshots.KindCommitted,
|
||||
},
|
||||
"committed-2": {
|
||||
Name: "committed-2",
|
||||
Parent: "committed-1",
|
||||
Kind: snapshots.KindCommitted,
|
||||
},
|
||||
"active-1": {
|
||||
Name: "active-1",
|
||||
Parent: "",
|
||||
Kind: snapshots.KindActive,
|
||||
},
|
||||
"active-2": {
|
||||
Name: "active-2",
|
||||
Parent: "committed-1",
|
||||
Kind: snapshots.KindActive,
|
||||
},
|
||||
"active-3": {
|
||||
Name: "active-3",
|
||||
Parent: "committed-2",
|
||||
Kind: snapshots.KindActive,
|
||||
},
|
||||
"view-1": {
|
||||
Name: "view-1",
|
||||
Parent: "",
|
||||
Kind: snapshots.KindView,
|
||||
},
|
||||
"view-2": {
|
||||
Name: "view-2",
|
||||
Parent: "committed-2",
|
||||
Kind: snapshots.KindView,
|
||||
},
|
||||
}
|
||||
|
||||
func assertNotExist(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
assert.True(t, errdefs.IsNotFound(err), "got %+v", err)
|
||||
}
|
||||
|
||||
func assertNotActive(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
assert.True(t, errdefs.IsFailedPrecondition(err), "got %+v", err)
|
||||
}
|
||||
|
||||
func assertNotCommitted(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
assert.True(t, errdefs.IsInvalidArgument(err), "got %+v", err)
|
||||
}
|
||||
|
||||
func assertExist(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
assert.True(t, errdefs.IsAlreadyExists(err), "got %+v", err)
|
||||
}
|
||||
|
||||
func testGetInfo(ctx context.Context, t *testing.T, _ *MetaStore) {
|
||||
for key, expected := range baseInfo {
|
||||
_, info, _, err := GetInfo(ctx, key)
|
||||
assert.Nil(t, err, "on key %v", key)
|
||||
assert.Truef(t, cmp.Equal(expected, info, cmpSnapshotInfo), "on key %v", key)
|
||||
}
|
||||
}
|
||||
|
||||
// compare snapshot.Info Updated and Created fields by checking they are
|
||||
// within a threshold of time.Now()
|
||||
var cmpSnapshotInfo = cmp.FilterPath(
|
||||
func(path cmp.Path) bool {
|
||||
field := path.Last().String()
|
||||
return field == ".Created" || field == ".Updated"
|
||||
},
|
||||
cmp.Comparer(func(expected, actual time.Time) bool {
|
||||
// cmp.Options must be symmetric, so swap the args
|
||||
if actual.IsZero() {
|
||||
actual, expected = expected, actual
|
||||
}
|
||||
if !expected.IsZero() {
|
||||
return false
|
||||
}
|
||||
// actual value should be within a few seconds of now
|
||||
now := time.Now()
|
||||
delta := now.Sub(actual)
|
||||
threshold := 30 * time.Second
|
||||
return delta > -threshold && delta < threshold
|
||||
}))
|
||||
|
||||
func testGetInfoNotExist(ctx context.Context, t *testing.T, _ *MetaStore) {
|
||||
_, _, _, err := GetInfo(ctx, "active-not-exist")
|
||||
assertNotExist(t, err)
|
||||
}
|
||||
|
||||
func testWalk(ctx context.Context, t *testing.T, _ *MetaStore) {
|
||||
found := map[string]snapshots.Info{}
|
||||
err := WalkInfo(ctx, func(ctx context.Context, info snapshots.Info) error {
|
||||
if _, ok := found[info.Name]; ok {
|
||||
return errors.New("entry already encountered")
|
||||
}
|
||||
found[info.Name] = info
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, cmp.Equal(baseInfo, found, cmpSnapshotInfo))
|
||||
}
|
||||
|
||||
func testGetSnapshot(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
snapshotMap := map[string]Snapshot{}
|
||||
populate := func(ctx context.Context, ms *MetaStore) error {
|
||||
if _, err := CreateSnapshot(ctx, snapshots.KindActive, "committed-tmp-1", ""); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1", snapshots.Usage{}); err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
|
||||
for _, opts := range []struct {
|
||||
Kind snapshots.Kind
|
||||
Name string
|
||||
Parent string
|
||||
}{
|
||||
{
|
||||
Name: "active-1",
|
||||
Kind: snapshots.KindActive,
|
||||
},
|
||||
{
|
||||
Name: "active-2",
|
||||
Parent: "committed-1",
|
||||
Kind: snapshots.KindActive,
|
||||
},
|
||||
{
|
||||
Name: "view-1",
|
||||
Kind: snapshots.KindView,
|
||||
},
|
||||
{
|
||||
Name: "view-2",
|
||||
Parent: "committed-1",
|
||||
Kind: snapshots.KindView,
|
||||
},
|
||||
} {
|
||||
active, err := CreateSnapshot(ctx, opts.Kind, opts.Name, opts.Parent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create active: %w", err)
|
||||
}
|
||||
snapshotMap[opts.Name] = active
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
test := func(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
for key, expected := range snapshotMap {
|
||||
s, err := GetSnapshot(ctx, key)
|
||||
assert.Nil(t, err, "failed to get snapshot %s", key)
|
||||
assert.Equalf(t, expected, s, "on key %s", key)
|
||||
}
|
||||
}
|
||||
|
||||
inReadTransaction(test, populate)(ctx, t, ms)
|
||||
}
|
||||
|
||||
func testGetSnapshotCommitted(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
_, err := GetSnapshot(ctx, "committed-1")
|
||||
assertNotActive(t, err)
|
||||
}
|
||||
|
||||
func testGetSnapshotNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
_, err := GetSnapshot(ctx, "active-not-exist")
|
||||
assertNotExist(t, err)
|
||||
}
|
||||
|
||||
func testCreateActive(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
a1, err := CreateSnapshot(ctx, snapshots.KindActive, "active-1", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a1.Kind != snapshots.KindActive {
|
||||
t.Fatal("Expected writable active")
|
||||
}
|
||||
|
||||
a2, err := CreateSnapshot(ctx, snapshots.KindView, "view-1", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a2.ID == a1.ID {
|
||||
t.Fatal("Returned active identifiers must be unique")
|
||||
}
|
||||
if a2.Kind != snapshots.KindView {
|
||||
t.Fatal("Expected a view")
|
||||
}
|
||||
|
||||
commitID, err := CommitActive(ctx, "active-1", "committed-1", snapshots.Usage{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if commitID != a1.ID {
|
||||
t.Fatal("Snapshot identifier must not change on commit")
|
||||
}
|
||||
|
||||
a3, err := CreateSnapshot(ctx, snapshots.KindActive, "active-3", "committed-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a3.ID == a1.ID {
|
||||
t.Fatal("Returned active identifiers must be unique")
|
||||
}
|
||||
if len(a3.ParentIDs) != 1 {
|
||||
t.Fatalf("Expected 1 parent, got %d", len(a3.ParentIDs))
|
||||
}
|
||||
if a3.ParentIDs[0] != commitID {
|
||||
t.Fatal("Expected active parent to be same as commit ID")
|
||||
}
|
||||
if a3.Kind != snapshots.KindActive {
|
||||
t.Fatal("Expected writable active")
|
||||
}
|
||||
|
||||
a4, err := CreateSnapshot(ctx, snapshots.KindView, "view-2", "committed-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a4.ID == a1.ID {
|
||||
t.Fatal("Returned active identifiers must be unique")
|
||||
}
|
||||
if len(a3.ParentIDs) != 1 {
|
||||
t.Fatalf("Expected 1 parent, got %d", len(a3.ParentIDs))
|
||||
}
|
||||
if a3.ParentIDs[0] != commitID {
|
||||
t.Fatal("Expected active parent to be same as commit ID")
|
||||
}
|
||||
if a4.Kind != snapshots.KindView {
|
||||
t.Fatal("Expected a view")
|
||||
}
|
||||
}
|
||||
|
||||
func testCreateActiveExist(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if err := basePopulate(ctx, ms); err != nil {
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
_, err := CreateSnapshot(ctx, snapshots.KindActive, "active-1", "")
|
||||
assertExist(t, err)
|
||||
_, err = CreateSnapshot(ctx, snapshots.KindActive, "committed-1", "")
|
||||
assertExist(t, err)
|
||||
}
|
||||
|
||||
func testCreateActiveNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
_, err := CreateSnapshot(ctx, snapshots.KindActive, "active-1", "does-not-exist")
|
||||
assertNotExist(t, err)
|
||||
}
|
||||
|
||||
func testCreateActiveFromActive(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if err := basePopulate(ctx, ms); err != nil {
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
_, err := CreateSnapshot(ctx, snapshots.KindActive, "active-new", "active-1")
|
||||
assertNotCommitted(t, err)
|
||||
}
|
||||
|
||||
func testCommit(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
a1, err := CreateSnapshot(ctx, snapshots.KindActive, "active-1", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a1.Kind != snapshots.KindActive {
|
||||
t.Fatal("Expected writable active")
|
||||
}
|
||||
|
||||
commitID, err := CommitActive(ctx, "active-1", "committed-1", snapshots.Usage{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if commitID != a1.ID {
|
||||
t.Fatal("Snapshot identifier must not change on commit")
|
||||
}
|
||||
|
||||
_, err = GetSnapshot(ctx, "active-1")
|
||||
assertNotExist(t, err)
|
||||
_, err = GetSnapshot(ctx, "committed-1")
|
||||
assertNotActive(t, err)
|
||||
}
|
||||
|
||||
func testCommitExist(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if err := basePopulate(ctx, ms); err != nil {
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
_, err := CommitActive(ctx, "active-1", "committed-1", snapshots.Usage{})
|
||||
assertExist(t, err)
|
||||
}
|
||||
|
||||
func testCommitCommitted(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if err := basePopulate(ctx, ms); err != nil {
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
_, err := CommitActive(ctx, "committed-1", "committed-3", snapshots.Usage{})
|
||||
assertNotActive(t, err)
|
||||
}
|
||||
|
||||
func testCommitViewFails(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if err := basePopulate(ctx, ms); err != nil {
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
_, err := CommitActive(ctx, "view-1", "committed-3", snapshots.Usage{})
|
||||
if err == nil {
|
||||
t.Fatal("Expected error committing readonly active")
|
||||
}
|
||||
}
|
||||
|
||||
func testRemove(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
a1, err := CreateSnapshot(ctx, snapshots.KindActive, "active-1", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
commitID, err := CommitActive(ctx, "active-1", "committed-1", snapshots.Usage{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if commitID != a1.ID {
|
||||
t.Fatal("Snapshot identifier must not change on commit")
|
||||
}
|
||||
|
||||
a2, err := CreateSnapshot(ctx, snapshots.KindView, "view-1", "committed-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a3, err := CreateSnapshot(ctx, snapshots.KindView, "view-2", "committed-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, _, err = Remove(ctx, "active-1")
|
||||
assertNotExist(t, err)
|
||||
|
||||
r3, k3, err := Remove(ctx, "view-2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r3 != a3.ID {
|
||||
t.Fatal("Expected remove ID to match create ID")
|
||||
}
|
||||
if k3 != snapshots.KindView {
|
||||
t.Fatalf("Expected view kind, got %v", k3)
|
||||
}
|
||||
|
||||
r2, k2, err := Remove(ctx, "view-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r2 != a2.ID {
|
||||
t.Fatal("Expected remove ID to match create ID")
|
||||
}
|
||||
if k2 != snapshots.KindView {
|
||||
t.Fatalf("Expected view kind, got %v", k2)
|
||||
}
|
||||
|
||||
r1, k1, err := Remove(ctx, "committed-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r1 != commitID {
|
||||
t.Fatal("Expected remove ID to match commit ID")
|
||||
}
|
||||
if k1 != snapshots.KindCommitted {
|
||||
t.Fatalf("Expected committed kind, got %v", k1)
|
||||
}
|
||||
}
|
||||
|
||||
func testRemoveWithChildren(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if err := basePopulate(ctx, ms); err != nil {
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
_, _, err := Remove(ctx, "committed-1")
|
||||
if err == nil {
|
||||
t.Fatalf("Expected removal of snapshot with children to error")
|
||||
}
|
||||
_, _, err = Remove(ctx, "committed-1")
|
||||
if err == nil {
|
||||
t.Fatalf("Expected removal of snapshot with children to error")
|
||||
}
|
||||
}
|
||||
|
||||
func testRemoveNotExist(ctx context.Context, t *testing.T, _ *MetaStore) {
|
||||
_, _, err := Remove(ctx, "does-not-exist")
|
||||
assertNotExist(t, err)
|
||||
}
|
||||
|
||||
func testParents(ctx context.Context, t *testing.T, ms *MetaStore) {
|
||||
if err := basePopulate(ctx, ms); err != nil {
|
||||
t.Fatalf("Populate failed: %+v", err)
|
||||
}
|
||||
|
||||
testcases := []struct {
|
||||
Name string
|
||||
Parents int
|
||||
}{
|
||||
{"committed-1", 0},
|
||||
{"committed-2", 1},
|
||||
{"active-1", 0},
|
||||
{"active-2", 1},
|
||||
{"active-3", 2},
|
||||
{"view-1", 0},
|
||||
{"view-2", 2},
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
name := tc.Name
|
||||
expectedID := ""
|
||||
expectedParents := []string{}
|
||||
for i := tc.Parents; i >= 0; i-- {
|
||||
sid, info, _, err := GetInfo(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get snapshot %s: %v", tc.Name, err)
|
||||
}
|
||||
var (
|
||||
id string
|
||||
parents []string
|
||||
)
|
||||
if info.Kind == snapshots.KindCommitted {
|
||||
// When committed, create view and resolve from view
|
||||
nid := fmt.Sprintf("test-%s-%d", tc.Name, i)
|
||||
s, err := CreateSnapshot(ctx, snapshots.KindView, nid, name)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get snapshot %s: %v", tc.Name, err)
|
||||
}
|
||||
if len(s.ParentIDs) != i+1 {
|
||||
t.Fatalf("Unexpected number of parents for view of %s: %d, expected %d", name, len(s.ParentIDs), i+1)
|
||||
}
|
||||
id = s.ParentIDs[0]
|
||||
parents = s.ParentIDs[1:]
|
||||
} else {
|
||||
s, err := GetSnapshot(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get snapshot %s: %v", tc.Name, err)
|
||||
}
|
||||
if len(s.ParentIDs) != i {
|
||||
t.Fatalf("Unexpected number of parents for %s: %d, expected %d", name, len(s.ParentIDs), i)
|
||||
}
|
||||
|
||||
id = s.ID
|
||||
parents = s.ParentIDs
|
||||
}
|
||||
if sid != id {
|
||||
t.Fatalf("Info ID mismatched resolved snapshot ID for %s, %s vs %s", name, sid, id)
|
||||
}
|
||||
|
||||
if expectedID != "" {
|
||||
if id != expectedID {
|
||||
t.Errorf("Unexpected ID of parent: %s, expected %s", id, expectedID)
|
||||
}
|
||||
}
|
||||
|
||||
if len(expectedParents) > 0 {
|
||||
for j := range expectedParents {
|
||||
if parents[j] != expectedParents[j] {
|
||||
t.Errorf("Unexpected ID in parent array at %d: %s, expected %s", j, parents[j], expectedParents[j])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if i > 0 {
|
||||
name = info.Parent
|
||||
expectedID = parents[0]
|
||||
expectedParents = parents[1:]
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user