Add snapshotters benchmark

Signed-off-by: Maksym Pavlenko <makpav@amazon.com>

snapshots/testsuite/benchmark_aws.yml

@@ -0,0 +1,144 @@
+AWSTemplateFormatVersion: "2010-09-09"
+
+Description: >
+  This templates spin ups an EC2 instance with EBS volumes suitable for containerd snapshotters benchmarking.
+  The template will create EBS volumes for benchmarking (/dev/sdb, /dev/sdc, and /dev/sdd) with same performance characteristics.
+  /dev/sde volume will be created and used for device mapper thin-pool device.
+
+Parameters:
+  Key:
+    Type: AWS::EC2::KeyPair::KeyName
+    Description: SSH key to use
+
+  AMI:
+    Type: AWS::EC2::Image::Id
+    Description: AMI ID to use for the EC2 instance. Must be Amazon Linux 2.
+    Default: "ami-032509850cf9ee54e"
+
+  SecurityGroups:
+    Type: List<AWS::EC2::SecurityGroup::Id>
+    Description: List of security groups to add to EC2 instance
+
+  InstanceType:
+    Type: String
+    Default: m4.xlarge
+    Description: EC2 instance type to use
+
+  VolumesIOPS:
+    Type: Number
+    Default: 1000
+    MinValue: 100
+    MaxValue: 20000
+    Description: The number of I/O operations per second (IOPS) to reserve for EBS volumes.
+
+  VolumesSize:
+    Type: Number
+    Default: 20
+    MinValue: 4
+    MaxValue: 16384
+    Description: EBS volumes size, in gibibytes (GiB)
+
+  VolumeType:
+    Type: String
+    Default: io1
+    AllowedValues:
+      - io1
+      - gp2
+      - sc1
+      - st1
+    Description: >
+      Volume type to use for EBS volumes (io1 is recommended).
+      More information on volume types https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html
+
+  ContainerStorageSetup:
+    Type: String
+    Default: https://github.com/projectatomic/container-storage-setup/archive/v0.6.0.tar.gz
+    Description: container-storage-setup tool version to install (more details at https://github.com/projectatomic/container-storage-setup)
+
+Resources:
+  Instance:
+    Type: AWS::EC2::Instance
+    Properties:
+      EbsOptimized: true
+      InstanceType: !Ref InstanceType
+      KeyName: !Ref Key
+      ImageId: !Ref AMI
+      SecurityGroupIds: !Ref SecurityGroups
+      BlockDeviceMappings:
+        - DeviceName: "/dev/xvda" # Root volume
+          Ebs:
+            VolumeSize: 64
+            VolumeType: io1
+            Iops: 1000
+            DeleteOnTermination: true
+        - DeviceName: "/dev/sdb"
+          Ebs:
+            VolumeSize: !Ref VolumesSize
+            VolumeType: !Ref VolumeType
+            Iops: !Ref VolumesIOPS
+            DeleteOnTermination: true
+        - DeviceName: "/dev/sdc"
+          Ebs:
+            VolumeSize: !Ref VolumesSize
+            VolumeType: !Ref VolumeType
+            Iops: !Ref VolumesIOPS
+            DeleteOnTermination: true
+        - DeviceName: "/dev/sdd"
+          Ebs:
+            VolumeSize: !Ref VolumesSize
+            VolumeType: !Ref VolumeType
+            Iops: !Ref VolumesIOPS
+            DeleteOnTermination: true
+        - DeviceName: "/dev/sde"
+          Ebs:
+            VolumeSize: !Ref VolumesSize
+            VolumeType: !Ref VolumeType
+            Iops: !Ref VolumesIOPS
+            DeleteOnTermination: true
+
+      UserData:
+        Fn::Base64:
+          !Sub |
+          #!/bin/bash
+
+          set -ex
+
+          yum install -y \
+            gcc \
+            git \
+            btrfs-progs-devel \
+            libseccomp-devel
+
+          amazon-linux-extras install -y golang1.11
+
+          # Install container-storage-setup
+          mkdir -p /tmp/container-storage-setup/unpacked/
+          cd /tmp/container-storage-setup/
+          curl -sL ${ContainerStorageSetup} -o archive.tar.gz
+          tar -xzf archive.tar.gz -C unpacked --strip 1
+          cd unpacked/
+          make install-core
+          rm -rf /tmp/container-storage-setup/
+
+          # Prepare EBS volumes
+          mkdir -p /mnt/{disk1,disk2,disk3}
+
+          mkfs.ext4 /dev/sdb
+          mount /dev/sdb /mnt/disk1/
+
+          mkfs.ext4 /dev/sdc
+          mount /dev/sdc /mnt/disk2/
+
+          mkfs.ext4 /dev/sdd
+          mount /dev/sdd /mnt/disk3
+
+          chgrp -R wheel /mnt/disk1/ /mnt/disk2/ /mnt/disk3/
+          chmod -R 2775 /mnt/disk1/ /mnt/disk2/ /mnt/disk3/
+
+          # Prepare thin-pool device
+          touch /etc/sysconfig/docker-storage-setup
+          echo DEVS=/dev/sde >> /etc/sysconfig/docker-storage-setup
+          echo VG=bench >> /etc/sysconfig/docker-storage-setup
+          container-storage-setup
+
+          echo "Done"

snapshots/testsuite/benchmark_readme.md

@@ -0,0 +1,83 @@
+## Requirements
+
+### Running
+Due to its dependency on `dmsetup`, executing the snapshotter process in an environment where a udev
+daemon is not accessible (such as a container) may result in unexpected behavior. In this case, try executing the
+snapshotter with the `DM_DISABLE_UDEV=1` environment variable, which tells `dmsetup` to ignore udev and manage devices
+itself. See [lvm(8)](http://man7.org/linux/man-pages/man8/lvm.8.html) and
+[dmsetup(8)](http://man7.org/linux/man-pages/man8/dmsetup.8.html) for more information.
+
+## How to run snapshotters benchmark
+
+- `containerd` project contains AWS CloudFormation template to run an EC2 instance suitable for benchmarking.
+It installs dependencies, prepares EBS volumes with same performance characteristics, and creates thin-pool device.
+You can make stack with the following command (note: there is a charge for using AWS resources):
+
+```bash
+aws cloudformation create-stack \
+    --stack-name benchmark-instance \
+    --template-body file://benchmark_aws.yml \
+    --parameters \
+        ParameterKey=Key,ParameterValue=SSH_KEY \
+        ParameterKey=SecurityGroups,ParameterValue=sg-XXXXXXXX \
+        ParameterKey=VolumesSize,ParameterValue=20 \
+        ParameterKey=VolumesIOPS,ParameterValue=1000
+```
+
+- You can find an IP address of newly created EC2 instance in AWS Console or via AWS CLI:
+
+```bash
+$ aws ec2 describe-instances \
+    --instance-ids $(aws cloudformation describe-stack-resources --stack-name benchmark-instance --query 'StackResources[*].PhysicalResourceId' --output text) \
+    --query 'Reservations[*].Instances[*].PublicIpAddress' \
+    --output text
+```
+
+- SSH to an instance and prepare `containerd`:
+
+```bash
+ssh -i SSH_KEY ec2-user@IP
+mkdir /mnt/disk1/data /mnt/disk2/data /mnt/disk3/data
+go get github.com/containerd/containerd
+cd $(go env GOPATH)/src/github.com/containerd/containerd
+make
+```
+
+- Now you're ready to run the benchmark:
+
+```bash
+sudo su -
+cd snapshots/testsuite/
+go test -bench . \
+    -dm.thinPoolDev=bench-docker--pool \
+    -dm.rootPath=/mnt/disk1/data \
+    -overlay.rootPath=/mnt/disk2/data \
+    -native.rootPath=/mnt/disk3/data
+```
+
+- The output will look like:
+
+```bash
+goos: linux
+goarch: amd64
+pkg: github.com/containerd/containerd/snapshots/testsuite
+
+BenchmarkOverlay/run-4             1       1019730210 ns/op	 164.53 MB/s
+BenchmarkOverlay/prepare           1         26799447 ns/op
+BenchmarkOverlay/write             1        968200363 ns/op
+BenchmarkOverlay/commit            1         24582560 ns/op
+
+BenchmarkDeviceMapper/run-4        1       3139232730 ns/op	  53.44 MB/s
+BenchmarkDeviceMapper/prepare	   1       1758640440 ns/op
+BenchmarkDeviceMapper/write        1       1356705388 ns/op
+BenchmarkDeviceMapper/commit       1         23720367 ns/op
+
+PASS
+ok  	github.com/containerd/containerd/snapshots/testsuite	185.204s
+```
+
+- Don't forget to tear down the stack so it does not continue to incur charges:
+
+```bash
+aws cloudformation delete-stack --stack-name benchmark-instance
+```

snapshots/testsuite/benchmark_test.go

@@ -0,0 +1,312 @@
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testsuite
+
+import (
+	"context"
+	"crypto/rand"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/containerd/continuity/fs/fstest"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"gotest.tools/assert"
+
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/snapshots"
+	"github.com/containerd/containerd/snapshots/devmapper"
+	"github.com/containerd/containerd/snapshots/native"
+	"github.com/containerd/containerd/snapshots/overlay"
+)
+
+var (
+	dmPoolDev       string
+	dmRootPath      string
+	overlayRootPath string
+	nativeRootPath  string
+)
+
+func init() {
+	flag.StringVar(&dmPoolDev, "dm.thinPoolDev", "", "Pool device to run benchmark on")
+	flag.StringVar(&dmRootPath, "dm.rootPath", "", "Root dir for devmapper snapshotter")
+	flag.StringVar(&overlayRootPath, "overlay.rootPath", "", "Root dir for overlay snapshotter")
+	flag.StringVar(&nativeRootPath, "native.rootPath", "", "Root dir for native snapshotter")
+
+	// Avoid mixing benchmark output and INFO messages
+	logrus.SetLevel(logrus.ErrorLevel)
+}
+
+func BenchmarkNative(b *testing.B) {
+	if nativeRootPath == "" {
+		b.Skip("native root dir must be provided")
+	}
+
+	snapshotter, err := native.NewSnapshotter(nativeRootPath)
+	assert.NilError(b, err)
+
+	defer func() {
+		err = snapshotter.Close()
+		assert.NilError(b, err)
+
+		err = os.RemoveAll(nativeRootPath)
+		assert.NilError(b, err)
+	}()
+
+	benchmarkSnapshotter(b, snapshotter)
+}
+
+func BenchmarkOverlay(b *testing.B) {
+	if overlayRootPath == "" {
+		b.Skip("overlay root dir must be provided")
+	}
+
+	snapshotter, err := overlay.NewSnapshotter(overlayRootPath)
+	assert.NilError(b, err, "failed to create overlay snapshotter")
+
+	defer func() {
+		err = snapshotter.Close()
+		assert.NilError(b, err)
+
+		err = os.RemoveAll(overlayRootPath)
+		assert.NilError(b, err)
+	}()
+
+	benchmarkSnapshotter(b, snapshotter)
+}
+
+func BenchmarkDeviceMapper(b *testing.B) {
+	if dmPoolDev == "" {
+		b.Skip("devmapper benchmark requires thin-pool device to be prepared in advance and provided")
+	}
+
+	if dmRootPath == "" {
+		b.Skip("devmapper snapshotter root dir must be provided")
+	}
+
+	config := &devmapper.Config{
+		PoolName:      dmPoolDev,
+		RootPath:      dmRootPath,
+		BaseImageSize: "16Mb",
+	}
+
+	ctx := context.Background()
+
+	snapshotter, err := devmapper.NewSnapshotter(ctx, config)
+	assert.NilError(b, err)
+
+	defer func() {
+		err := snapshotter.ResetPool(ctx)
+		assert.NilError(b, err)
+
+		err = snapshotter.Close()
+		assert.NilError(b, err)
+
+		err = os.RemoveAll(dmRootPath)
+		assert.NilError(b, err)
+	}()
+
+	benchmarkSnapshotter(b, snapshotter)
+}
+
+// benchmarkSnapshotter tests snapshotter performance.
+// It writes 16 layers with randomly created, modified, or removed files.
+// Depending on layer index different sets of files are modified.
+// In addition to total snapshotter execution time, benchmark outputs a few additional
+// details - time taken to Prepare layer, mount, write data and unmount time,
+// and Commit snapshot time.
+func benchmarkSnapshotter(b *testing.B, snapshotter snapshots.Snapshotter) {
+	const (
+		layerCount    = 16
+		fileSizeBytes = int64(1 * 1024 * 1024) // 1 MB
+	)
+
+	var (
+		total      = 0
+		layers     = make([]fstest.Applier, 0, layerCount)
+		layerIndex = int64(0)
+	)
+
+	for i := 1; i <= layerCount; i++ {
+		appliers := makeApplier(i, fileSizeBytes)
+		layers = append(layers, fstest.Apply(appliers...))
+		total += len(appliers)
+	}
+
+	var (
+		benchN          int
+		prepareDuration time.Duration
+		writeDuration   time.Duration
+		commitDuration  time.Duration
+	)
+
+	// Wrap test with Run so additional details output will be added right below the benchmark result
+	b.Run("run", func(b *testing.B) {
+		var (
+			ctx     = context.Background()
+			parent  string
+			current string
+		)
+
+		// Reset durations since test might be ran multiple times
+		prepareDuration = 0
+		writeDuration = 0
+		commitDuration = 0
+		benchN = b.N
+
+		b.SetBytes(int64(total) * fileSizeBytes)
+
+		var timer time.Time
+		for i := 0; i < b.N; i++ {
+			for l := 0; l < layerCount; l++ {
+				current = fmt.Sprintf("prepare-layer-%d", atomic.AddInt64(&layerIndex, 1))
+
+				timer = time.Now()
+				mounts, err := snapshotter.Prepare(ctx, current, parent)
+				assert.NilError(b, err)
+				prepareDuration += time.Since(timer)
+
+				timer = time.Now()
+				err = mount.WithTempMount(ctx, mounts, layers[l].Apply)
+				assert.NilError(b, err)
+				writeDuration += time.Since(timer)
+
+				parent = fmt.Sprintf("comitted-%d", atomic.AddInt64(&layerIndex, 1))
+
+				timer = time.Now()
+				err = snapshotter.Commit(ctx, parent, current)
+				assert.NilError(b, err)
+				commitDuration += time.Since(timer)
+			}
+		}
+	})
+
+	// Output extra measurements - total time taken to Prepare, mount and write data, and Commit
+	const outputFormat = "%-25s\t%s\n"
+	fmt.Fprintf(os.Stdout,
+		outputFormat,
+		b.Name()+"/prepare",
+		testing.BenchmarkResult{N: benchN, T: prepareDuration})
+
+	fmt.Fprintf(os.Stdout,
+		outputFormat,
+		b.Name()+"/write",
+		testing.BenchmarkResult{N: benchN, T: writeDuration})
+
+	fmt.Fprintf(os.Stdout,
+		outputFormat,
+		b.Name()+"/commit",
+		testing.BenchmarkResult{N: benchN, T: commitDuration})
+
+	fmt.Fprintln(os.Stdout)
+}
+
+// makeApplier returns a slice of fstest.Applier where files are written randomly.
+// Depending on layer index, the returned layers will overwrite some files with the
+// same generated names with new contents or deletions.
+func makeApplier(layerIndex int, fileSizeBytes int64) []fstest.Applier {
+	seed := time.Now().UnixNano()
+
+	switch {
+	case layerIndex%3 == 0:
+		return []fstest.Applier{
+			updateFile("/a"),
+			updateFile("/b"),
+			fstest.CreateRandomFile("/c", seed, fileSizeBytes, 0777),
+			updateFile("/d"),
+			fstest.CreateRandomFile("/f", seed, fileSizeBytes, 0777),
+			updateFile("/e"),
+			fstest.RemoveAll("/g"),
+			fstest.CreateRandomFile("/h", seed, fileSizeBytes, 0777),
+			updateFile("/i"),
+			fstest.CreateRandomFile("/j", seed, fileSizeBytes, 0777),
+		}
+	case layerIndex%2 == 0:
+		return []fstest.Applier{
+			updateFile("/a"),
+			fstest.CreateRandomFile("/b", seed, fileSizeBytes, 0777),
+			fstest.RemoveAll("/c"),
+			fstest.CreateRandomFile("/d", seed, fileSizeBytes, 0777),
+			updateFile("/e"),
+			fstest.RemoveAll("/f"),
+			fstest.CreateRandomFile("/g", seed, fileSizeBytes, 0777),
+			updateFile("/h"),
+			fstest.CreateRandomFile("/i", seed, fileSizeBytes, 0777),
+			updateFile("/j"),
+		}
+	default:
+		return []fstest.Applier{
+			fstest.CreateRandomFile("/a", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/b", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/c", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/d", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/e", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/f", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/g", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/h", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/i", seed, fileSizeBytes, 0777),
+			fstest.CreateRandomFile("/j", seed, fileSizeBytes, 0777),
+		}
+	}
+}
+
+// applierFn represents helper func that implements fstest.Applier
+type applierFn func(root string) error
+
+func (fn applierFn) Apply(root string) error {
+	return fn(root)
+}
+
+// updateFile modifies a few bytes in the middle in order to demonstrate the difference in performance
+// for block-based snapshotters (like devicemapper) against file-based snapshotters (like overlay, which need to
+// perform a copy-up of the full file any time a single bit is modified).
+func updateFile(name string) applierFn {
+	return func(root string) error {
+		path := filepath.Join(root, name)
+		file, err := os.OpenFile(path, os.O_WRONLY, 0600)
+		if err != nil {
+			return errors.Wrapf(err, "failed to open %q", path)
+		}
+
+		info, err := file.Stat()
+		if err != nil {
+			return err
+		}
+
+		var (
+			offset = info.Size() / 2
+			buf    = make([]byte, 4)
+		)
+
+		if _, err := rand.Read(buf); err != nil {
+			return err
+		}
+
+		if _, err := file.WriteAt(buf, offset); err != nil {
+			return errors.Wrapf(err, "failed to write %q at offset %d", path, offset)
+		}
+
+		return file.Close()
+	}
+}