diff --git a/contrib/aws/snapshotter_bench_cf.yml b/contrib/aws/snapshotter_bench_cf.yml new file mode 100644 index 000000000..5883494bc --- /dev/null +++ b/contrib/aws/snapshotter_bench_cf.yml @@ -0,0 +1,144 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Description: > + This templates spin ups an EC2 instance with EBS volumes suitable for containerd snapshotters benchmarking. + The template will create EBS volumes for benchmarking (/dev/sdb, /dev/sdc, and /dev/sdd) with same performance characteristics. + /dev/sde volume will be created and used for device mapper thin-pool device. + +Parameters: + Key: + Type: AWS::EC2::KeyPair::KeyName + Description: SSH key to use + + AMI: + Type: AWS::EC2::Image::Id + Description: AMI ID to use for the EC2 instance. Must be Amazon Linux 2. + Default: "ami-032509850cf9ee54e" + + SecurityGroups: + Type: List + Description: List of security groups to add to EC2 instance + + InstanceType: + Type: String + Default: m4.xlarge + Description: EC2 instance type to use + + VolumesIOPS: + Type: Number + Default: 1000 + MinValue: 100 + MaxValue: 20000 + Description: The number of I/O operations per second (IOPS) to reserve for EBS volumes. + + VolumesSize: + Type: Number + Default: 20 + MinValue: 4 + MaxValue: 16384 + Description: EBS volumes size, in gibibytes (GiB) + + VolumeType: + Type: String + Default: io1 + AllowedValues: + - io1 + - gp2 + - sc1 + - st1 + Description: > + Volume type to use for EBS volumes (io1 is recommended). + More information on volume types https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html + + ContainerStorageSetup: + Type: String + Default: https://github.com/projectatomic/container-storage-setup/archive/v0.6.0.tar.gz + Description: container-storage-setup tool version to install (more details at https://github.com/projectatomic/container-storage-setup) + +Resources: + Instance: + Type: AWS::EC2::Instance + Properties: + EbsOptimized: true + InstanceType: !Ref InstanceType + KeyName: !Ref Key + ImageId: !Ref AMI + SecurityGroupIds: !Ref SecurityGroups + BlockDeviceMappings: + - DeviceName: "/dev/xvda" # Root volume + Ebs: + VolumeSize: 64 + VolumeType: io1 + Iops: 1000 + DeleteOnTermination: true + - DeviceName: "/dev/sdb" + Ebs: + VolumeSize: !Ref VolumesSize + VolumeType: !Ref VolumeType + Iops: !Ref VolumesIOPS + DeleteOnTermination: true + - DeviceName: "/dev/sdc" + Ebs: + VolumeSize: !Ref VolumesSize + VolumeType: !Ref VolumeType + Iops: !Ref VolumesIOPS + DeleteOnTermination: true + - DeviceName: "/dev/sdd" + Ebs: + VolumeSize: !Ref VolumesSize + VolumeType: !Ref VolumeType + Iops: !Ref VolumesIOPS + DeleteOnTermination: true + - DeviceName: "/dev/sde" + Ebs: + VolumeSize: !Ref VolumesSize + VolumeType: !Ref VolumeType + Iops: !Ref VolumesIOPS + DeleteOnTermination: true + + UserData: + Fn::Base64: + !Sub | + #!/bin/bash + + set -ex + + yum install -y \ + gcc \ + git \ + btrfs-progs-devel \ + libseccomp-devel + + amazon-linux-extras install -y golang1.11 + + # Install container-storage-setup + mkdir -p /tmp/container-storage-setup/unpacked/ + cd /tmp/container-storage-setup/ + curl -sL ${ContainerStorageSetup} -o archive.tar.gz + tar -xzf archive.tar.gz -C unpacked --strip 1 + cd unpacked/ + make install-core + rm -rf /tmp/container-storage-setup/ + + # Prepare EBS volumes + mkdir -p /mnt/{disk1,disk2,disk3} + + mkfs.ext4 /dev/sdb + mount /dev/sdb /mnt/disk1/ + + mkfs.ext4 /dev/sdc + mount /dev/sdc /mnt/disk2/ + + mkfs.ext4 /dev/sdd + mount /dev/sdd /mnt/disk3 + + chgrp -R wheel /mnt/disk1/ /mnt/disk2/ /mnt/disk3/ + chmod -R 2775 /mnt/disk1/ /mnt/disk2/ /mnt/disk3/ + + # Prepare thin-pool device + touch /etc/sysconfig/docker-storage-setup + echo DEVS=/dev/sde >> /etc/sysconfig/docker-storage-setup + echo VG=bench >> /etc/sysconfig/docker-storage-setup + container-storage-setup + + echo "Done" diff --git a/contrib/aws/snapshotter_bench_readme.md b/contrib/aws/snapshotter_bench_readme.md new file mode 100644 index 000000000..8d1049bde --- /dev/null +++ b/contrib/aws/snapshotter_bench_readme.md @@ -0,0 +1,83 @@ +## Requirements + +### Running +Due to its dependency on `dmsetup`, executing the snapshotter process in an environment where a udev +daemon is not accessible (such as a container) may result in unexpected behavior. In this case, try executing the +snapshotter with the `DM_DISABLE_UDEV=1` environment variable, which tells `dmsetup` to ignore udev and manage devices +itself. See [lvm(8)](http://man7.org/linux/man-pages/man8/lvm.8.html) and +[dmsetup(8)](http://man7.org/linux/man-pages/man8/dmsetup.8.html) for more information. + +## How to run snapshotters benchmark + +- `containerd` project contains AWS CloudFormation template to run an EC2 instance suitable for benchmarking. +It installs dependencies, prepares EBS volumes with same performance characteristics, and creates thin-pool device. +You can make stack with the following command (note: there is a charge for using AWS resources): + +```bash +aws cloudformation create-stack \ + --stack-name benchmark-instance \ + --template-body file://benchmark_aws.yml \ + --parameters \ + ParameterKey=Key,ParameterValue=SSH_KEY \ + ParameterKey=SecurityGroups,ParameterValue=sg-XXXXXXXX \ + ParameterKey=VolumesSize,ParameterValue=20 \ + ParameterKey=VolumesIOPS,ParameterValue=1000 +``` + +- You can find an IP address of newly created EC2 instance in AWS Console or via AWS CLI: + +```bash +$ aws ec2 describe-instances \ + --instance-ids $(aws cloudformation describe-stack-resources --stack-name benchmark-instance --query 'StackResources[*].PhysicalResourceId' --output text) \ + --query 'Reservations[*].Instances[*].PublicIpAddress' \ + --output text +``` + +- SSH to an instance and prepare `containerd`: + +```bash +ssh -i SSH_KEY ec2-user@IP +mkdir /mnt/disk1/data /mnt/disk2/data /mnt/disk3/data +go get github.com/containerd/containerd +cd $(go env GOPATH)/src/github.com/containerd/containerd +make +``` + +- Now you're ready to run the benchmark: + +```bash +sudo su - +cd snapshots/testsuite/ +go test -bench . \ + -dm.thinPoolDev=bench-docker--pool \ + -dm.rootPath=/mnt/disk1/data \ + -overlay.rootPath=/mnt/disk2/data \ + -native.rootPath=/mnt/disk3/data +``` + +- The output will look like: + +```bash +goos: linux +goarch: amd64 +pkg: github.com/containerd/containerd/snapshots/testsuite + +BenchmarkOverlay/run-4 1 1019730210 ns/op 164.53 MB/s +BenchmarkOverlay/prepare 1 26799447 ns/op +BenchmarkOverlay/write 1 968200363 ns/op +BenchmarkOverlay/commit 1 24582560 ns/op + +BenchmarkDeviceMapper/run-4 1 3139232730 ns/op 53.44 MB/s +BenchmarkDeviceMapper/prepare 1 1758640440 ns/op +BenchmarkDeviceMapper/write 1 1356705388 ns/op +BenchmarkDeviceMapper/commit 1 23720367 ns/op + +PASS +ok github.com/containerd/containerd/snapshots/testsuite 185.204s +``` + +- Don't forget to tear down the stack so it does not continue to incur charges: + +```bash +aws cloudformation delete-stack --stack-name benchmark-instance +``` diff --git a/snapshots/testsuite/benchmark_test.go b/snapshots/testsuite/benchmark_test.go new file mode 100644 index 000000000..d4894a329 --- /dev/null +++ b/snapshots/testsuite/benchmark_test.go @@ -0,0 +1,312 @@ +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testsuite + +import ( + "context" + "crypto/rand" + "flag" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/containerd/continuity/fs/fstest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "gotest.tools/assert" + + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/devmapper" + "github.com/containerd/containerd/snapshots/native" + "github.com/containerd/containerd/snapshots/overlay" +) + +var ( + dmPoolDev string + dmRootPath string + overlayRootPath string + nativeRootPath string +) + +func init() { + flag.StringVar(&dmPoolDev, "dm.thinPoolDev", "", "Pool device to run benchmark on") + flag.StringVar(&dmRootPath, "dm.rootPath", "", "Root dir for devmapper snapshotter") + flag.StringVar(&overlayRootPath, "overlay.rootPath", "", "Root dir for overlay snapshotter") + flag.StringVar(&nativeRootPath, "native.rootPath", "", "Root dir for native snapshotter") + + // Avoid mixing benchmark output and INFO messages + logrus.SetLevel(logrus.ErrorLevel) +} + +func BenchmarkNative(b *testing.B) { + if nativeRootPath == "" { + b.Skip("native root dir must be provided") + } + + snapshotter, err := native.NewSnapshotter(nativeRootPath) + assert.NilError(b, err) + + defer func() { + err = snapshotter.Close() + assert.NilError(b, err) + + err = os.RemoveAll(nativeRootPath) + assert.NilError(b, err) + }() + + benchmarkSnapshotter(b, snapshotter) +} + +func BenchmarkOverlay(b *testing.B) { + if overlayRootPath == "" { + b.Skip("overlay root dir must be provided") + } + + snapshotter, err := overlay.NewSnapshotter(overlayRootPath) + assert.NilError(b, err, "failed to create overlay snapshotter") + + defer func() { + err = snapshotter.Close() + assert.NilError(b, err) + + err = os.RemoveAll(overlayRootPath) + assert.NilError(b, err) + }() + + benchmarkSnapshotter(b, snapshotter) +} + +func BenchmarkDeviceMapper(b *testing.B) { + if dmPoolDev == "" { + b.Skip("devmapper benchmark requires thin-pool device to be prepared in advance and provided") + } + + if dmRootPath == "" { + b.Skip("devmapper snapshotter root dir must be provided") + } + + config := &devmapper.Config{ + PoolName: dmPoolDev, + RootPath: dmRootPath, + BaseImageSize: "16Mb", + } + + ctx := context.Background() + + snapshotter, err := devmapper.NewSnapshotter(ctx, config) + assert.NilError(b, err) + + defer func() { + err := snapshotter.ResetPool(ctx) + assert.NilError(b, err) + + err = snapshotter.Close() + assert.NilError(b, err) + + err = os.RemoveAll(dmRootPath) + assert.NilError(b, err) + }() + + benchmarkSnapshotter(b, snapshotter) +} + +// benchmarkSnapshotter tests snapshotter performance. +// It writes 16 layers with randomly created, modified, or removed files. +// Depending on layer index different sets of files are modified. +// In addition to total snapshotter execution time, benchmark outputs a few additional +// details - time taken to Prepare layer, mount, write data and unmount time, +// and Commit snapshot time. +func benchmarkSnapshotter(b *testing.B, snapshotter snapshots.Snapshotter) { + const ( + layerCount = 16 + fileSizeBytes = int64(1 * 1024 * 1024) // 1 MB + ) + + var ( + total = 0 + layers = make([]fstest.Applier, 0, layerCount) + layerIndex = int64(0) + ) + + for i := 1; i <= layerCount; i++ { + appliers := makeApplier(i, fileSizeBytes) + layers = append(layers, fstest.Apply(appliers...)) + total += len(appliers) + } + + var ( + benchN int + prepareDuration time.Duration + writeDuration time.Duration + commitDuration time.Duration + ) + + // Wrap test with Run so additional details output will be added right below the benchmark result + b.Run("run", func(b *testing.B) { + var ( + ctx = context.Background() + parent string + current string + ) + + // Reset durations since test might be ran multiple times + prepareDuration = 0 + writeDuration = 0 + commitDuration = 0 + benchN = b.N + + b.SetBytes(int64(total) * fileSizeBytes) + + var timer time.Time + for i := 0; i < b.N; i++ { + for l := 0; l < layerCount; l++ { + current = fmt.Sprintf("prepare-layer-%d", atomic.AddInt64(&layerIndex, 1)) + + timer = time.Now() + mounts, err := snapshotter.Prepare(ctx, current, parent) + assert.NilError(b, err) + prepareDuration += time.Since(timer) + + timer = time.Now() + err = mount.WithTempMount(ctx, mounts, layers[l].Apply) + assert.NilError(b, err) + writeDuration += time.Since(timer) + + parent = fmt.Sprintf("comitted-%d", atomic.AddInt64(&layerIndex, 1)) + + timer = time.Now() + err = snapshotter.Commit(ctx, parent, current) + assert.NilError(b, err) + commitDuration += time.Since(timer) + } + } + }) + + // Output extra measurements - total time taken to Prepare, mount and write data, and Commit + const outputFormat = "%-25s\t%s\n" + fmt.Fprintf(os.Stdout, + outputFormat, + b.Name()+"/prepare", + testing.BenchmarkResult{N: benchN, T: prepareDuration}) + + fmt.Fprintf(os.Stdout, + outputFormat, + b.Name()+"/write", + testing.BenchmarkResult{N: benchN, T: writeDuration}) + + fmt.Fprintf(os.Stdout, + outputFormat, + b.Name()+"/commit", + testing.BenchmarkResult{N: benchN, T: commitDuration}) + + fmt.Fprintln(os.Stdout) +} + +// makeApplier returns a slice of fstest.Applier where files are written randomly. +// Depending on layer index, the returned layers will overwrite some files with the +// same generated names with new contents or deletions. +func makeApplier(layerIndex int, fileSizeBytes int64) []fstest.Applier { + seed := time.Now().UnixNano() + + switch { + case layerIndex%3 == 0: + return []fstest.Applier{ + updateFile("/a"), + updateFile("/b"), + fstest.CreateRandomFile("/c", seed, fileSizeBytes, 0777), + updateFile("/d"), + fstest.CreateRandomFile("/f", seed, fileSizeBytes, 0777), + updateFile("/e"), + fstest.RemoveAll("/g"), + fstest.CreateRandomFile("/h", seed, fileSizeBytes, 0777), + updateFile("/i"), + fstest.CreateRandomFile("/j", seed, fileSizeBytes, 0777), + } + case layerIndex%2 == 0: + return []fstest.Applier{ + updateFile("/a"), + fstest.CreateRandomFile("/b", seed, fileSizeBytes, 0777), + fstest.RemoveAll("/c"), + fstest.CreateRandomFile("/d", seed, fileSizeBytes, 0777), + updateFile("/e"), + fstest.RemoveAll("/f"), + fstest.CreateRandomFile("/g", seed, fileSizeBytes, 0777), + updateFile("/h"), + fstest.CreateRandomFile("/i", seed, fileSizeBytes, 0777), + updateFile("/j"), + } + default: + return []fstest.Applier{ + fstest.CreateRandomFile("/a", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/b", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/c", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/d", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/e", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/f", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/g", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/h", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/i", seed, fileSizeBytes, 0777), + fstest.CreateRandomFile("/j", seed, fileSizeBytes, 0777), + } + } +} + +// applierFn represents helper func that implements fstest.Applier +type applierFn func(root string) error + +func (fn applierFn) Apply(root string) error { + return fn(root) +} + +// updateFile modifies a few bytes in the middle in order to demonstrate the difference in performance +// for block-based snapshotters (like devicemapper) against file-based snapshotters (like overlay, which need to +// perform a copy-up of the full file any time a single bit is modified). +func updateFile(name string) applierFn { + return func(root string) error { + path := filepath.Join(root, name) + file, err := os.OpenFile(path, os.O_WRONLY, 0600) + if err != nil { + return errors.Wrapf(err, "failed to open %q", path) + } + + info, err := file.Stat() + if err != nil { + return err + } + + var ( + offset = info.Size() / 2 + buf = make([]byte, 4) + ) + + if _, err := rand.Read(buf); err != nil { + return err + } + + if _, err := file.WriteAt(buf, offset); err != nil { + return errors.Wrapf(err, "failed to write %q at offset %d", path, offset) + } + + return file.Close() + } +}