kubernetes/test/e2e/storage/utils/ebs.go

/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"fmt"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/klog/v2"
)

const (
	volumeAttachmentStatusPollDelay = 2 * time.Second
	volumeAttachmentStatusFactor    = 2
	volumeAttachmentStatusSteps     = 6

	// represents expected attachment status of a volume after attach
	volumeAttachedStatus = "attached"

	// represents expected attachment status of a volume after detach
	volumeDetachedStatus = "detached"
)

// EBSUtil provides functions to interact with EBS volumes
type EBSUtil struct {
	client       *ec2.EC2
	validDevices []string
}

// NewEBSUtil returns an instance of EBSUtil which can be used to
// to interact with EBS volumes
func NewEBSUtil(client *ec2.EC2) *EBSUtil {
	ebsUtil := &EBSUtil{client: client}
	validDevices := []string{}
	for _, firstChar := range []rune{'b', 'c'} {
		for i := 'a'; i <= 'z'; i++ {
			dev := string([]rune{firstChar, i})
			validDevices = append(validDevices, dev)
		}
	}
	ebsUtil.validDevices = validDevices
	return ebsUtil
}

// AttachDisk attaches an EBS volume to a node.
func (ebs *EBSUtil) AttachDisk(volumeID string, nodeName string) error {
	instance, err := findInstanceByNodeName(nodeName, ebs.client)
	if err != nil {
		return fmt.Errorf("error finding node %s: %w", nodeName, err)
	}
	err = ebs.waitForAvailable(volumeID)
	if err != nil {
		return fmt.Errorf("error waiting volume %s to be available: %w", volumeID, err)
	}

	device, err := ebs.findFreeDevice(instance)
	if err != nil {
		return fmt.Errorf("error finding free device on node %s: %w", nodeName, err)
	}
	hostDevice := "/dev/xvd" + string(device)
	attachInput := &ec2.AttachVolumeInput{
		VolumeId:   &volumeID,
		InstanceId: instance.InstanceId,
		Device:     &hostDevice,
	}
	_, err = ebs.client.AttachVolume(attachInput)
	if err != nil {
		return fmt.Errorf("error attaching volume %s to node %s: %w", volumeID, nodeName, err)
	}
	return ebs.waitForAttach(volumeID)
}

func (ebs *EBSUtil) findFreeDevice(instance *ec2.Instance) (string, error) {
	deviceMappings := map[string]string{}

	for _, blockDevice := range instance.BlockDeviceMappings {
		name := aws.StringValue(blockDevice.DeviceName)
		name = strings.TrimPrefix(name, "/dev/sd")
		name = strings.TrimPrefix(name, "/dev/xvd")
		if len(name) < 1 || len(name) > 2 {
			klog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName))
		}

		deviceMappings[name] = aws.StringValue(blockDevice.Ebs.VolumeId)
	}

	for _, device := range ebs.validDevices {
		if _, found := deviceMappings[device]; !found {
			return device, nil
		}
	}
	return "", fmt.Errorf("no available device")
}

func (ebs *EBSUtil) waitForAttach(volumeID string) error {
	backoff := wait.Backoff{
		Duration: volumeAttachmentStatusPollDelay,
		Factor:   volumeAttachmentStatusFactor,
		Steps:    volumeAttachmentStatusSteps,
	}
	time.Sleep(volumeAttachmentStatusPollDelay)
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		info, err := ebs.describeVolume(volumeID)
		if err != nil {
			return false, err
		}

		if len(info.Attachments) > 1 {
			// Shouldn't happen; log so we know if it is
			klog.Warningf("Found multiple attachments for volume %q: %v", volumeID, info)
		}
		attachmentStatus := ""
		for _, a := range info.Attachments {
			if attachmentStatus != "" {
				// Shouldn't happen; log so we know if it is
				klog.Warningf("Found multiple attachments for volume %q: %v", volumeID, info)
			}
			if a.State != nil {
				attachmentStatus = *a.State
			} else {
				// Shouldn't happen; log so we know if it is
				klog.Warningf("Ignoring nil attachment state for volume %q: %v", volumeID, a)
			}
		}
		if attachmentStatus == "" {
			attachmentStatus = volumeDetachedStatus
		}
		if attachmentStatus == volumeAttachedStatus {
			// Attachment is in requested state, finish waiting
			return true, nil
		}
		return false, nil
	})
	return err
}

func (ebs *EBSUtil) waitForAvailable(volumeID string) error {
	backoff := wait.Backoff{
		Duration: volumeAttachmentStatusPollDelay,
		Factor:   volumeAttachmentStatusFactor,
		Steps:    volumeAttachmentStatusSteps,
	}
	time.Sleep(volumeAttachmentStatusPollDelay)
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		info, err := ebs.describeVolume(volumeID)
		if err != nil {
			return false, err
		}
		volumeState := aws.StringValue(info.State)
		if volumeState != ec2.VolumeStateAvailable {
			return false, nil
		}
		return true, nil
	})
	return err
}

// Gets the full information about this volume from the EC2 API
func (ebs *EBSUtil) describeVolume(volumeID string) (*ec2.Volume, error) {
	request := &ec2.DescribeVolumesInput{
		VolumeIds: []*string{&volumeID},
	}

	results := []*ec2.Volume{}
	var nextToken *string
	for {
		response, err := ebs.client.DescribeVolumes(request)
		if err != nil {
			return nil, err
		}

		results = append(results, response.Volumes...)

		nextToken = response.NextToken
		if aws.StringValue(nextToken) == "" {
			break
		}
		request.NextToken = nextToken
	}

	if len(results) == 0 {
		return nil, fmt.Errorf("no volumes found")
	}
	if len(results) > 1 {
		return nil, fmt.Errorf("multiple volumes found")
	}
	return results[0], nil
}

func newEc2Filter(name string, value string) *ec2.Filter {
	filter := &ec2.Filter{
		Name: aws.String(name),
		Values: []*string{
			aws.String(value),
		},
	}
	return filter
}

func findInstanceByNodeName(nodeName string, cloud *ec2.EC2) (*ec2.Instance, error) {
	filters := []*ec2.Filter{
		newEc2Filter("private-dns-name", nodeName),
	}

	request := &ec2.DescribeInstancesInput{
		Filters: filters,
	}

	instances, err := describeInstances(request, cloud)
	if err != nil {
		return nil, err
	}
	if len(instances) == 0 {
		return nil, nil
	}
	if len(instances) > 1 {
		return nil, fmt.Errorf("multiple instances found for name: %s", nodeName)
	}
	return instances[0], nil
}

func describeInstances(request *ec2.DescribeInstancesInput, cloud *ec2.EC2) ([]*ec2.Instance, error) {
	// Instances are paged
	results := []*ec2.Instance{}
	var nextToken *string

	for {
		response, err := cloud.DescribeInstances(request)
		if err != nil {
			return nil, fmt.Errorf("error listing AWS instances: %w", err)
		}

		for _, reservation := range response.Reservations {
			results = append(results, reservation.Instances...)
		}

		nextToken = response.NextToken
		if nextToken == nil || len(*nextToken) == 0 {
			break
		}
		request.NextToken = nextToken
	}

	return results, nil
}