From cec72efc2a98c3fa5d698a3bd041113729745ddb Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 14 Feb 2019 13:48:13 -0800 Subject: [PATCH] devmapper: add snapshotter Signed-off-by: Maksym Pavlenko --- snapshots/devmapper/snapshotter.go | 392 ++++++++++++++++++++++++ snapshots/devmapper/snapshotter_test.go | 78 +++++ 2 files changed, 470 insertions(+) create mode 100644 snapshots/devmapper/snapshotter.go create mode 100644 snapshots/devmapper/snapshotter_test.go diff --git a/snapshots/devmapper/snapshotter.go b/snapshots/devmapper/snapshotter.go new file mode 100644 index 000000000..aa396c4e9 --- /dev/null +++ b/snapshots/devmapper/snapshotter.go @@ -0,0 +1,392 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package devmapper + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/devmapper/dmsetup" + "github.com/containerd/containerd/snapshots/storage" + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + metadataFileName = "metadata.db" + fsTypeExt4 = "ext4" +) + +type closeFunc func() error + +// Snapshotter implements containerd's snapshotter (https://godoc.org/github.com/containerd/containerd/snapshots#Snapshotter) +// based on Linux device-mapper targets. +type Snapshotter struct { + store *storage.MetaStore + pool *PoolDevice + config *Config + cleanupFn []closeFunc + closeOnce sync.Once +} + +// NewSnapshotter creates new device mapper snapshotter. +// Internally it creates thin-pool device (or reloads if it's already exists) and +// initializes a database file for metadata. +func NewSnapshotter(ctx context.Context, config *Config) (*Snapshotter, error) { + // Make sure snapshotter configuration valid before running + if err := config.parse(); err != nil { + return nil, err + } + + if err := config.Validate(); err != nil { + return nil, err + } + + var cleanupFn []closeFunc + + if err := os.MkdirAll(config.RootPath, 0750); err != nil && !os.IsExist(err) { + return nil, errors.Wrapf(err, "failed to create root directory: %s", config.RootPath) + } + + store, err := storage.NewMetaStore(filepath.Join(config.RootPath, metadataFileName)) + if err != nil { + return nil, errors.Wrap(err, "failed to create metastore") + } + + cleanupFn = append(cleanupFn, store.Close) + + poolDevice, err := NewPoolDevice(ctx, config) + if err != nil { + return nil, err + } + + cleanupFn = append(cleanupFn, poolDevice.Close) + + return &Snapshotter{ + store: store, + config: config, + pool: poolDevice, + cleanupFn: cleanupFn, + }, nil +} + +// Stat returns the info for an active or committed snapshot from store +func (s *Snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { + log.G(ctx).WithField("key", key).Debug("stat") + + var ( + info snapshots.Info + err error + ) + + err = s.withTransaction(ctx, false, func(ctx context.Context) error { + _, info, _, err = storage.GetInfo(ctx, key) + return err + }) + + return info, err +} + +// Update updates an existing snapshot info's data +func (s *Snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { + log.G(ctx).Debugf("update: %s", strings.Join(fieldpaths, ", ")) + + var err error + err = s.withTransaction(ctx, true, func(ctx context.Context) error { + info, err = storage.UpdateInfo(ctx, info, fieldpaths...) + return err + }) + + return info, err +} + +// Usage not yet implemented +func (s *Snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { + log.G(ctx).WithField("key", key).Debug("usage") + + return snapshots.Usage{}, errors.New("usage not implemented") +} + +// Mounts return the list of mounts for the active or view snapshot +func (s *Snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) { + log.G(ctx).WithField("key", key).Debug("mounts") + + var ( + snap storage.Snapshot + err error + ) + + err = s.withTransaction(ctx, false, func(ctx context.Context) error { + snap, err = storage.GetSnapshot(ctx, key) + return err + }) + + return s.buildMounts(snap), nil +} + +// Prepare creates thin device for an active snapshot identified by key +func (s *Snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + log.G(ctx).WithFields(logrus.Fields{"key": key, "parent": parent}).Debug("prepare") + + var ( + mounts []mount.Mount + err error + ) + + err = s.withTransaction(ctx, true, func(ctx context.Context) error { + mounts, err = s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts...) + return err + }) + + return mounts, err +} + +// View creates readonly thin device for the given snapshot key +func (s *Snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + log.G(ctx).WithFields(logrus.Fields{"key": key, "parent": parent}).Debug("prepare") + + var ( + mounts []mount.Mount + err error + ) + + err = s.withTransaction(ctx, true, func(ctx context.Context) error { + mounts, err = s.createSnapshot(ctx, snapshots.KindView, key, parent, opts...) + return err + }) + + return mounts, err +} + +// Commit marks an active snapshot as committed in meta store. +// Block device unmount operation captures snapshot changes by itself, so no +// additional actions needed within Commit operation. +func (s *Snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { + log.G(ctx).WithFields(logrus.Fields{"name": name, "key": key}).Debug("commit") + + return s.withTransaction(ctx, true, func(ctx context.Context) error { + _, err := storage.CommitActive(ctx, key, name, snapshots.Usage{}, opts...) + return err + }) +} + +// Remove removes thin device and snapshot metadata by key +func (s *Snapshotter) Remove(ctx context.Context, key string) error { + log.G(ctx).WithField("key", key).Debug("remove") + + return s.withTransaction(ctx, true, func(ctx context.Context) error { + return s.removeDevice(ctx, key) + }) +} + +func (s *Snapshotter) removeDevice(ctx context.Context, key string) error { + snapID, _, err := storage.Remove(ctx, key) + if err != nil { + return err + } + + deviceName := s.getDeviceName(snapID) + if err := s.pool.RemoveDevice(ctx, deviceName); err != nil { + log.G(ctx).WithError(err).Errorf("failed to remove device") + return err + } + + return nil +} + +// Walk iterates through all metadata Info for the stored snapshots and calls the provided function for each. +func (s *Snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error { + log.G(ctx).Debug("walk") + return s.withTransaction(ctx, false, func(ctx context.Context) error { + return storage.WalkInfo(ctx, fn) + }) +} + +// ResetPool deactivates and deletes all thin devices in thin-pool. +// Used for cleaning pool after benchmarking. +func (s *Snapshotter) ResetPool(ctx context.Context) error { + names, err := s.pool.metadata.GetDeviceNames(ctx) + if err != nil { + return err + } + + var result *multierror.Error + for _, name := range names { + if err := s.pool.RemoveDevice(ctx, name); err != nil { + result = multierror.Append(result, err) + } + } + + return result.ErrorOrNil() +} + +// Close releases devmapper snapshotter resources. +// All subsequent Close calls will be ignored. +func (s *Snapshotter) Close() error { + log.L.Debug("close") + + var result *multierror.Error + s.closeOnce.Do(func() { + for _, fn := range s.cleanupFn { + if err := fn(); err != nil { + result = multierror.Append(result, err) + } + } + }) + + return result.ErrorOrNil() +} + +func (s *Snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + snap, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...) + if err != nil { + return nil, err + } + + if len(snap.ParentIDs) == 0 { + deviceName := s.getDeviceName(snap.ID) + log.G(ctx).Debugf("creating new thin device '%s'", deviceName) + + err := s.pool.CreateThinDevice(ctx, deviceName, s.config.BaseImageSizeBytes) + if err != nil { + log.G(ctx).WithError(err).Errorf("failed to create thin device for snapshot %s", snap.ID) + return nil, err + } + + if err := s.mkfs(ctx, deviceName); err != nil { + return nil, err + } + } else { + parentDeviceName := s.getDeviceName(snap.ParentIDs[0]) + snapDeviceName := s.getDeviceName(snap.ID) + log.G(ctx).Debugf("creating snapshot device '%s' from '%s'", snapDeviceName, parentDeviceName) + + err := s.pool.CreateSnapshotDevice(ctx, parentDeviceName, snapDeviceName, s.config.BaseImageSizeBytes) + if err != nil { + log.G(ctx).WithError(err).Errorf("failed to create snapshot device from parent %s", parentDeviceName) + return nil, err + } + } + + mounts := s.buildMounts(snap) + + // Remove default directories not expected by the container image + _ = mount.WithTempMount(ctx, mounts, func(root string) error { + return os.Remove(filepath.Join(root, "lost+found")) + }) + + return mounts, nil +} + +// mkfs creates ext4 filesystem on the given devmapper device +func (s *Snapshotter) mkfs(ctx context.Context, deviceName string) error { + args := []string{ + "-E", + // We don't want any zeroing in advance when running mkfs on thin devices (see "man mkfs.ext4") + "nodiscard,lazy_itable_init=0,lazy_journal_init=0", + dmsetup.GetFullDevicePath(deviceName), + } + + log.G(ctx).Debugf("mkfs.ext4 %s", strings.Join(args, " ")) + output, err := exec.Command("mkfs.ext4", args...).CombinedOutput() + if err != nil { + log.G(ctx).WithError(err).Errorf("failed to write fs:\n%s", string(output)) + return err + } + + log.G(ctx).Debugf("mkfs:\n%s", string(output)) + return nil +} + +func (s *Snapshotter) getDeviceName(snapID string) string { + // Add pool name as prefix to avoid collisions with devices from other pools + return fmt.Sprintf("%s-snap-%s", s.config.PoolName, snapID) +} + +func (s *Snapshotter) getDevicePath(snap storage.Snapshot) string { + name := s.getDeviceName(snap.ID) + return dmsetup.GetFullDevicePath(name) +} + +func (s *Snapshotter) buildMounts(snap storage.Snapshot) []mount.Mount { + var options []string + + if snap.Kind != snapshots.KindActive { + options = append(options, "ro") + } + + mounts := []mount.Mount{ + { + Source: s.getDevicePath(snap), + Type: fsTypeExt4, + Options: options, + }, + } + + return mounts +} + +// withTransaction wraps fn callback with containerd's meta store transaction. +// If callback returns an error or transaction is not writable, database transaction will be discarded. +func (s *Snapshotter) withTransaction(ctx context.Context, writable bool, fn func(ctx context.Context) error) error { + ctx, trans, err := s.store.TransactionContext(ctx, writable) + if err != nil { + return err + } + + var result *multierror.Error + + err = fn(ctx) + if err != nil { + result = multierror.Append(result, err) + } + + // Always rollback if transaction is not writable + if err != nil || !writable { + if terr := trans.Rollback(); terr != nil { + log.G(ctx).WithError(terr).Error("failed to rollback transaction") + result = multierror.Append(result, errors.Wrap(terr, "rollback failed")) + } + } else { + if terr := trans.Commit(); terr != nil { + log.G(ctx).WithError(terr).Error("failed to commit transaction") + result = multierror.Append(result, errors.Wrap(terr, "commit failed")) + } + } + + if err := result.ErrorOrNil(); err != nil { + log.G(ctx).WithError(err).Debug("snapshotter error") + + // Unwrap if just one error + if len(result.Errors) == 1 { + return result.Errors[0] + } + + return err + } + + return nil +} diff --git a/snapshots/devmapper/snapshotter_test.go b/snapshots/devmapper/snapshotter_test.go new file mode 100644 index 000000000..148bf3a60 --- /dev/null +++ b/snapshots/devmapper/snapshotter_test.go @@ -0,0 +1,78 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package devmapper + +import ( + "context" + _ "crypto/sha256" + "fmt" + "testing" + "time" + + "github.com/containerd/containerd/pkg/testutil" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/devmapper/dmsetup" + "github.com/containerd/containerd/snapshots/devmapper/losetup" + "github.com/containerd/containerd/snapshots/testsuite" + "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestSnapshotterSuite(t *testing.T) { + testutil.RequiresRoot(t) + + logrus.SetLevel(logrus.DebugLevel) + + testsuite.SnapshotterSuite(t, "devmapper", func(ctx context.Context, root string) (snapshots.Snapshotter, func() error, error) { + // Create loopback devices for each test case + _, loopDataDevice := createLoopbackDevice(t, root) + _, loopMetaDevice := createLoopbackDevice(t, root) + + poolName := fmt.Sprintf("containerd-snapshotter-suite-pool-%d", time.Now().Nanosecond()) + err := dmsetup.CreatePool(poolName, loopDataDevice, loopMetaDevice, 64*1024/dmsetup.SectorSize) + assert.NilError(t, err, "failed to create pool %q", poolName) + + config := &Config{ + RootPath: root, + PoolName: poolName, + BaseImageSize: "16Mb", + } + + snap, err := NewSnapshotter(context.Background(), config) + if err != nil { + return nil, nil, err + } + + // Remove device mapper pool after test completes + removePool := func() error { + return snap.pool.RemovePool(ctx) + } + + // Pool cleanup should be called before closing metadata store (as we need to retrieve device names) + snap.cleanupFn = append([]closeFunc{removePool}, snap.cleanupFn...) + + return snap, func() error { + err := snap.Close() + assert.NilError(t, err, "failed to close snapshotter") + + err = losetup.DetachLoopDevice(loopDataDevice, loopMetaDevice) + assert.NilError(t, err, "failed to detach loop devices") + + return err + }, nil + }) +}