AWS: Enabling resize tests

pkg/cloudprovider/aws/aws.go

@@ -33,6 +33,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elb"
 
@@ -52,6 +53,7 @@ const TagNameKubernetesCluster = "KubernetesCluster"
 type AWSServices interface {
 	Compute(region string) (EC2, error)
 	LoadBalancing(region string) (ELB, error)
+	Autoscaling(region string) (ASG, error)
 	Metadata() AWSMetadata
 }
 
@@ -103,6 +105,12 @@ type ELB interface {
 	DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error)
 }
 
+// This is a simple pass-through of the Autoscaling client interface, which allows for testing
+type ASG interface {
+	UpdateAutoScalingGroup(*autoscaling.UpdateAutoScalingGroupInput) (*autoscaling.UpdateAutoScalingGroupOutput, error)
+	DescribeAutoScalingGroups(*autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error)
+}
+
 // Abstraction over the AWS metadata service
 type AWSMetadata interface {
 	// Query the EC2 metadata service (used to discover instance-id etc)
@@ -114,6 +122,7 @@ type VolumeOptions struct {
 }
 
 // Volumes is an interface for managing cloud-provisioned volumes
+// TODO: Allow other clouds to implement this
 type Volumes interface {
 	// Attach the disk to the specified instance
 	// instanceName can be empty to mean "the instance on which we are running"
@@ -128,10 +137,26 @@ type Volumes interface {
 	DeleteVolume(volumeName string) error
 }
 
+// InstanceGroups is an interface for managing cloud-managed instance groups / autoscaling instance groups
+// TODO: Allow other clouds to implement this
+type InstanceGroups interface {
+	// Set the size to the fixed size
+	ResizeInstanceGroup(instanceGroupName string, size int) error
+	// Queries the cloud provider for information about the specified instance group
+	DescribeInstanceGroup(instanceGroupName string) (InstanceGroupInfo, error)
+}
+
+// InstanceGroupInfo is returned by InstanceGroups.Describe, and exposes information about the group.
+type InstanceGroupInfo interface {
+	// The number of instances currently running under control of this group
+	CurrentSize() (int, error)
+}
+
 // AWSCloud is an implementation of Interface, TCPLoadBalancer and Instances for Amazon Web Services.
 type AWSCloud struct {
 	awsServices      AWSServices
 	ec2              EC2
+	asg              ASG
 	cfg              *AWSCloudConfig
 	availabilityZone string
 	region           string
@@ -183,6 +208,14 @@ func (p *awsSDKProvider) LoadBalancing(regionName string) (ELB, error) {
 	return elbClient, nil
 }
 
+func (p *awsSDKProvider) Autoscaling(regionName string) (ASG, error) {
+	client := autoscaling.New(&aws.Config{
+		Region:      regionName,
+		Credentials: p.creds,
+	})
+	return client, nil
+}
+
 func (p *awsSDKProvider) Metadata() AWSMetadata {
 	return &awsSdkMetadata{}
 }
@@ -512,10 +545,19 @@ func newAWSCloud(config io.Reader, awsServices AWSServices) (*AWSCloud, error) {
 	}
 
 	ec2, err := awsServices.Compute(regionName)
+	if err != nil {
+		return nil, fmt.Errorf("error creating AWS EC2 client: %v", err)
+	}
+
+	asg, err := awsServices.Autoscaling(regionName)
+	if err != nil {
+		return nil, fmt.Errorf("error creating AWS autoscaling client: %v", err)
+	}
 
 	awsCloud := &AWSCloud{
 		awsServices:      awsServices,
 		ec2:              ec2,
+		asg:              asg,
 		cfg:              cfg,
 		region:           regionName,
 		availabilityZone: zone,

pkg/cloudprovider/aws/aws_instancegroups.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws_cloud
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/golang/glog"
+)
+
+// AWSCloud implements InstanceGroups
+var _ InstanceGroups = &AWSCloud{}
+
+// Implement InstanceGroups.ResizeInstanceGroup
+// Set the size to the fixed size
+func (a *AWSCloud) ResizeInstanceGroup(instanceGroupName string, size int) error {
+	request := &autoscaling.UpdateAutoScalingGroupInput{
+		AutoScalingGroupName: aws.String(instanceGroupName),
+		MinSize:              aws.Long(int64(size)),
+		MaxSize:              aws.Long(int64(size)),
+	}
+	if _, err := a.asg.UpdateAutoScalingGroup(request); err != nil {
+		return fmt.Errorf("error resizing AWS autoscaling group: %v", err)
+	}
+	return nil
+}
+
+// Implement InstanceGroups.DescribeInstanceGroup
+// Queries the cloud provider for information about the specified instance group
+func (a *AWSCloud) DescribeInstanceGroup(instanceGroupName string) (InstanceGroupInfo, error) {
+	request := &autoscaling.DescribeAutoScalingGroupsInput{
+		AutoScalingGroupNames: []*string{aws.String(instanceGroupName)},
+	}
+	response, err := a.asg.DescribeAutoScalingGroups(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing AWS autoscaling group (%s): %v", instanceGroupName, err)
+	}
+
+	if len(response.AutoScalingGroups) == 0 {
+		return nil, nil
+	}
+	if len(response.AutoScalingGroups) > 1 {
+		glog.Warning("AWS returned multiple autoscaling groups with name ", instanceGroupName)
+	}
+	group := response.AutoScalingGroups[0]
+	return &awsInstanceGroup{group: group}, nil
+}
+
+// awsInstanceGroup implements InstanceGroupInfo
+var _ InstanceGroupInfo = &awsInstanceGroup{}
+
+type awsInstanceGroup struct {
+	group *autoscaling.Group
+}
+
+// Implement InstanceGroupInfo.CurrentSize
+// The number of instances currently running under control of this group
+func (g *awsInstanceGroup) CurrentSize() (int, error) {
+	return len(g.group.Instances), nil
+}

pkg/cloudprovider/aws/aws_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/golang/glog"
 )
 
@@ -110,6 +111,7 @@ type FakeAWSServices struct {
 
 	ec2      *FakeEC2
 	elb      *FakeELB
+	asg      *FakeASG
 	metadata *FakeMetadata
 }
 
@@ -118,6 +120,7 @@ func NewFakeAWSServices() *FakeAWSServices {
 	s.availabilityZone = "us-east-1a"
 	s.ec2 = &FakeEC2{aws: s}
 	s.elb = &FakeELB{aws: s}
+	s.asg = &FakeASG{aws: s}
 	s.metadata = &FakeMetadata{aws: s}
 
 	s.instanceId = "i-self"
@@ -151,6 +154,10 @@ func (s *FakeAWSServices) LoadBalancing(region string) (ELB, error) {
 	return s.elb, nil
 }
 
+func (s *FakeAWSServices) Autoscaling(region string) (ASG, error) {
+	return s.asg, nil
+}
+
 func (s *FakeAWSServices) Metadata() AWSMetadata {
 	return s.metadata
 }
@@ -396,6 +403,18 @@ func (ec2 *FakeELB) DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstances
 	panic("Not implemented")
 }
 
+type FakeASG struct {
+	aws *FakeAWSServices
+}
+
+func (a *FakeASG) UpdateAutoScalingGroup(*autoscaling.UpdateAutoScalingGroupInput) (*autoscaling.UpdateAutoScalingGroupOutput, error) {
+	panic("Not implemented")
+}
+
+func (a *FakeASG) DescribeAutoScalingGroups(*autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) {
+	panic("Not implemented")
+}
+
 func mockInstancesResp(instances []*ec2.Instance) *AWSCloud {
 	awsServices := NewFakeAWSServices().withInstances(instances)
 	return &AWSCloud{

test/e2e/e2e_test.go

@@ -90,7 +90,7 @@ func init() {
 	flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable")
 	flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable")
 	flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable")
-	flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce")
+	flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws")
 	flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster")
 
 	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")

test/e2e/resize_nodes.go

@@ -31,6 +31,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
 
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/aws"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -38,38 +39,65 @@ import (
 const serveHostnameImage = "gcr.io/google_containers/serve_hostname:1.1"
 
 func resizeGroup(size int) error {
-	// TODO: make this hit the compute API directly instread of shelling out to gcloud.
+	if testContext.Provider == "gce" || testContext.Provider == "gke" {
-	output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone,
+		// TODO: make this hit the compute API directly instread of shelling out to gcloud.
-		"resize", testContext.CloudConfig.NodeInstanceGroup, fmt.Sprintf("--new-size=%v", size)).CombinedOutput()
+		// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
-	if err != nil {
+		output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone,
-		Logf("Failed to resize node instance group: %v", string(output))
+			"resize", testContext.CloudConfig.NodeInstanceGroup, fmt.Sprintf("--new-size=%v", size)).CombinedOutput()
+		if err != nil {
+			Logf("Failed to resize node instance group: %v", string(output))
+		}
+		return err
+	} else {
+		// Supported by aws
+		instanceGroups, ok := testContext.CloudConfig.Provider.(aws_cloud.InstanceGroups)
+		if !ok {
+			return fmt.Errorf("Provider does not support InstanceGroups")
+		}
+		return instanceGroups.ResizeInstanceGroup(testContext.CloudConfig.NodeInstanceGroup, size)
 	}
-	return err
 }
 
 func groupSize() (int, error) {
-	// TODO: make this hit the compute API directly instread of shelling out to gcloud.
+	if testContext.Provider == "gce" || testContext.Provider == "gke" {
-	output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID,
+		// TODO: make this hit the compute API directly instread of shelling out to gcloud.
-		"--zone="+testContext.CloudConfig.Zone, "describe", testContext.CloudConfig.NodeInstanceGroup).CombinedOutput()
+		// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
-	if err != nil {
+		output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID,
-		return -1, err
+			"--zone="+testContext.CloudConfig.Zone, "describe", testContext.CloudConfig.NodeInstanceGroup).CombinedOutput()
-	}
+		if err != nil {
-	pattern := "currentSize: "
+			return -1, err
-	i := strings.Index(string(output), pattern)
+		}
-	if i == -1 {
+		pattern := "currentSize: "
-		return -1, fmt.Errorf("could not find '%s' in the output '%s'", pattern, output)
+		i := strings.Index(string(output), pattern)
-	}
+		if i == -1 {
-	truncated := output[i+len(pattern):]
+			return -1, fmt.Errorf("could not find '%s' in the output '%s'", pattern, output)
-	j := strings.Index(string(truncated), "\n")
+		}
-	if j == -1 {
+		truncated := output[i+len(pattern):]
-		return -1, fmt.Errorf("could not find new line in the truncated output '%s'", truncated)
+		j := strings.Index(string(truncated), "\n")
-	}
+		if j == -1 {
+			return -1, fmt.Errorf("could not find new line in the truncated output '%s'", truncated)
+		}
 
-	currentSize, err := strconv.Atoi(string(truncated[:j]))
+		currentSize, err := strconv.Atoi(string(truncated[:j]))
-	if err != nil {
+		if err != nil {
-		return -1, err
+			return -1, err
+		}
+		return currentSize, nil
+	} else {
+		// Supported by aws
+		instanceGroups, ok := testContext.CloudConfig.Provider.(aws_cloud.InstanceGroups)
+		if !ok {
+			return -1, fmt.Errorf("provider does not support InstanceGroups")
+		}
+		instanceGroup, err := instanceGroups.DescribeInstanceGroup(testContext.CloudConfig.NodeInstanceGroup)
+		if err != nil {
+			return -1, fmt.Errorf("error describing instance group: %v", err)
+		}
+		if instanceGroup == nil {
+			return -1, fmt.Errorf("instance group not found: %s", testContext.CloudConfig.NodeInstanceGroup)
+		}
+		return instanceGroup.CurrentSize()
 	}
-	return currentSize, nil
 }
 
 func waitForGroupSize(size int) error {
@@ -358,7 +386,7 @@ func performTemporaryNetworkFailure(c *client.Client, ns, rcName string, replica
 }
 
 var _ = Describe("Nodes", func() {
-	supportedProviders := []string{"gce", "gke"}
+	supportedProviders := []string{"aws", "gce", "gke"}
 	var testName string
 	var c *client.Client
 	var ns string