diff --git a/pkg/cloudprovider/aws/aws.go b/pkg/cloudprovider/aws/aws.go index 125d7fbcdd3..30d302f5050 100644 --- a/pkg/cloudprovider/aws/aws.go +++ b/pkg/cloudprovider/aws/aws.go @@ -33,6 +33,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" @@ -52,6 +53,7 @@ const TagNameKubernetesCluster = "KubernetesCluster" type AWSServices interface { Compute(region string) (EC2, error) LoadBalancing(region string) (ELB, error) + Autoscaling(region string) (ASG, error) Metadata() AWSMetadata } @@ -103,6 +105,12 @@ type ELB interface { DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error) } +// This is a simple pass-through of the Autoscaling client interface, which allows for testing +type ASG interface { + UpdateAutoScalingGroup(*autoscaling.UpdateAutoScalingGroupInput) (*autoscaling.UpdateAutoScalingGroupOutput, error) + DescribeAutoScalingGroups(*autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) +} + // Abstraction over the AWS metadata service type AWSMetadata interface { // Query the EC2 metadata service (used to discover instance-id etc) @@ -114,6 +122,7 @@ type VolumeOptions struct { } // Volumes is an interface for managing cloud-provisioned volumes +// TODO: Allow other clouds to implement this type Volumes interface { // Attach the disk to the specified instance // instanceName can be empty to mean "the instance on which we are running" @@ -128,10 +137,26 @@ type Volumes interface { DeleteVolume(volumeName string) error } +// InstanceGroups is an interface for managing cloud-managed instance groups / autoscaling instance groups +// TODO: Allow other clouds to implement this +type InstanceGroups interface { + // Set the size to the fixed size + ResizeInstanceGroup(instanceGroupName string, size int) error + // Queries the cloud provider for information about the specified instance group + DescribeInstanceGroup(instanceGroupName string) (InstanceGroupInfo, error) +} + +// InstanceGroupInfo is returned by InstanceGroups.Describe, and exposes information about the group. +type InstanceGroupInfo interface { + // The number of instances currently running under control of this group + CurrentSize() (int, error) +} + // AWSCloud is an implementation of Interface, TCPLoadBalancer and Instances for Amazon Web Services. type AWSCloud struct { awsServices AWSServices ec2 EC2 + asg ASG cfg *AWSCloudConfig availabilityZone string region string @@ -183,6 +208,14 @@ func (p *awsSDKProvider) LoadBalancing(regionName string) (ELB, error) { return elbClient, nil } +func (p *awsSDKProvider) Autoscaling(regionName string) (ASG, error) { + client := autoscaling.New(&aws.Config{ + Region: regionName, + Credentials: p.creds, + }) + return client, nil +} + func (p *awsSDKProvider) Metadata() AWSMetadata { return &awsSdkMetadata{} } @@ -512,10 +545,19 @@ func newAWSCloud(config io.Reader, awsServices AWSServices) (*AWSCloud, error) { } ec2, err := awsServices.Compute(regionName) + if err != nil { + return nil, fmt.Errorf("error creating AWS EC2 client: %v", err) + } + + asg, err := awsServices.Autoscaling(regionName) + if err != nil { + return nil, fmt.Errorf("error creating AWS autoscaling client: %v", err) + } awsCloud := &AWSCloud{ awsServices: awsServices, ec2: ec2, + asg: asg, cfg: cfg, region: regionName, availabilityZone: zone, diff --git a/pkg/cloudprovider/aws/aws_instancegroups.go b/pkg/cloudprovider/aws/aws_instancegroups.go new file mode 100644 index 00000000000..11d2ac6a74b --- /dev/null +++ b/pkg/cloudprovider/aws/aws_instancegroups.go @@ -0,0 +1,75 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws_cloud + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/golang/glog" +) + +// AWSCloud implements InstanceGroups +var _ InstanceGroups = &AWSCloud{} + +// Implement InstanceGroups.ResizeInstanceGroup +// Set the size to the fixed size +func (a *AWSCloud) ResizeInstanceGroup(instanceGroupName string, size int) error { + request := &autoscaling.UpdateAutoScalingGroupInput{ + AutoScalingGroupName: aws.String(instanceGroupName), + MinSize: aws.Long(int64(size)), + MaxSize: aws.Long(int64(size)), + } + if _, err := a.asg.UpdateAutoScalingGroup(request); err != nil { + return fmt.Errorf("error resizing AWS autoscaling group: %v", err) + } + return nil +} + +// Implement InstanceGroups.DescribeInstanceGroup +// Queries the cloud provider for information about the specified instance group +func (a *AWSCloud) DescribeInstanceGroup(instanceGroupName string) (InstanceGroupInfo, error) { + request := &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: []*string{aws.String(instanceGroupName)}, + } + response, err := a.asg.DescribeAutoScalingGroups(request) + if err != nil { + return nil, fmt.Errorf("error listing AWS autoscaling group (%s): %v", instanceGroupName, err) + } + + if len(response.AutoScalingGroups) == 0 { + return nil, nil + } + if len(response.AutoScalingGroups) > 1 { + glog.Warning("AWS returned multiple autoscaling groups with name ", instanceGroupName) + } + group := response.AutoScalingGroups[0] + return &awsInstanceGroup{group: group}, nil +} + +// awsInstanceGroup implements InstanceGroupInfo +var _ InstanceGroupInfo = &awsInstanceGroup{} + +type awsInstanceGroup struct { + group *autoscaling.Group +} + +// Implement InstanceGroupInfo.CurrentSize +// The number of instances currently running under control of this group +func (g *awsInstanceGroup) CurrentSize() (int, error) { + return len(g.group.Instances), nil +} diff --git a/pkg/cloudprovider/aws/aws_test.go b/pkg/cloudprovider/aws/aws_test.go index f9458be8796..5fd13daa4e7 100644 --- a/pkg/cloudprovider/aws/aws_test.go +++ b/pkg/cloudprovider/aws/aws_test.go @@ -28,6 +28,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource" + "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/golang/glog" ) @@ -110,6 +111,7 @@ type FakeAWSServices struct { ec2 *FakeEC2 elb *FakeELB + asg *FakeASG metadata *FakeMetadata } @@ -118,6 +120,7 @@ func NewFakeAWSServices() *FakeAWSServices { s.availabilityZone = "us-east-1a" s.ec2 = &FakeEC2{aws: s} s.elb = &FakeELB{aws: s} + s.asg = &FakeASG{aws: s} s.metadata = &FakeMetadata{aws: s} s.instanceId = "i-self" @@ -151,6 +154,10 @@ func (s *FakeAWSServices) LoadBalancing(region string) (ELB, error) { return s.elb, nil } +func (s *FakeAWSServices) Autoscaling(region string) (ASG, error) { + return s.asg, nil +} + func (s *FakeAWSServices) Metadata() AWSMetadata { return s.metadata } @@ -396,6 +403,18 @@ func (ec2 *FakeELB) DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstances panic("Not implemented") } +type FakeASG struct { + aws *FakeAWSServices +} + +func (a *FakeASG) UpdateAutoScalingGroup(*autoscaling.UpdateAutoScalingGroupInput) (*autoscaling.UpdateAutoScalingGroupOutput, error) { + panic("Not implemented") +} + +func (a *FakeASG) DescribeAutoScalingGroups(*autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) { + panic("Not implemented") +} + func mockInstancesResp(instances []*ec2.Instance) *AWSCloud { awsServices := NewFakeAWSServices().withInstances(instances) return &AWSCloud{ diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 98df0876b54..d203c997efe 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -90,7 +90,7 @@ func init() { flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable") flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable") flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable") - flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce") + flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws") flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster") flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources. Only required if provider is aws.") diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go index 94522f01571..360b10b8b38 100644 --- a/test/e2e/resize_nodes.go +++ b/test/e2e/resize_nodes.go @@ -31,6 +31,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait" + "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/aws" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -38,38 +39,65 @@ import ( const serveHostnameImage = "gcr.io/google_containers/serve_hostname:1.1" func resizeGroup(size int) error { - // TODO: make this hit the compute API directly instread of shelling out to gcloud. - output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone, - "resize", testContext.CloudConfig.NodeInstanceGroup, fmt.Sprintf("--new-size=%v", size)).CombinedOutput() - if err != nil { - Logf("Failed to resize node instance group: %v", string(output)) + if testContext.Provider == "gce" || testContext.Provider == "gke" { + // TODO: make this hit the compute API directly instread of shelling out to gcloud. + // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic + output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone, + "resize", testContext.CloudConfig.NodeInstanceGroup, fmt.Sprintf("--new-size=%v", size)).CombinedOutput() + if err != nil { + Logf("Failed to resize node instance group: %v", string(output)) + } + return err + } else { + // Supported by aws + instanceGroups, ok := testContext.CloudConfig.Provider.(aws_cloud.InstanceGroups) + if !ok { + return fmt.Errorf("Provider does not support InstanceGroups") + } + return instanceGroups.ResizeInstanceGroup(testContext.CloudConfig.NodeInstanceGroup, size) } - return err } func groupSize() (int, error) { - // TODO: make this hit the compute API directly instread of shelling out to gcloud. - output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID, - "--zone="+testContext.CloudConfig.Zone, "describe", testContext.CloudConfig.NodeInstanceGroup).CombinedOutput() - if err != nil { - return -1, err - } - pattern := "currentSize: " - i := strings.Index(string(output), pattern) - if i == -1 { - return -1, fmt.Errorf("could not find '%s' in the output '%s'", pattern, output) - } - truncated := output[i+len(pattern):] - j := strings.Index(string(truncated), "\n") - if j == -1 { - return -1, fmt.Errorf("could not find new line in the truncated output '%s'", truncated) - } + if testContext.Provider == "gce" || testContext.Provider == "gke" { + // TODO: make this hit the compute API directly instread of shelling out to gcloud. + // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic + output, err := exec.Command("gcloud", "preview", "managed-instance-groups", "--project="+testContext.CloudConfig.ProjectID, + "--zone="+testContext.CloudConfig.Zone, "describe", testContext.CloudConfig.NodeInstanceGroup).CombinedOutput() + if err != nil { + return -1, err + } + pattern := "currentSize: " + i := strings.Index(string(output), pattern) + if i == -1 { + return -1, fmt.Errorf("could not find '%s' in the output '%s'", pattern, output) + } + truncated := output[i+len(pattern):] + j := strings.Index(string(truncated), "\n") + if j == -1 { + return -1, fmt.Errorf("could not find new line in the truncated output '%s'", truncated) + } - currentSize, err := strconv.Atoi(string(truncated[:j])) - if err != nil { - return -1, err + currentSize, err := strconv.Atoi(string(truncated[:j])) + if err != nil { + return -1, err + } + return currentSize, nil + } else { + // Supported by aws + instanceGroups, ok := testContext.CloudConfig.Provider.(aws_cloud.InstanceGroups) + if !ok { + return -1, fmt.Errorf("provider does not support InstanceGroups") + } + instanceGroup, err := instanceGroups.DescribeInstanceGroup(testContext.CloudConfig.NodeInstanceGroup) + if err != nil { + return -1, fmt.Errorf("error describing instance group: %v", err) + } + if instanceGroup == nil { + return -1, fmt.Errorf("instance group not found: %s", testContext.CloudConfig.NodeInstanceGroup) + } + return instanceGroup.CurrentSize() } - return currentSize, nil } func waitForGroupSize(size int) error { @@ -358,7 +386,7 @@ func performTemporaryNetworkFailure(c *client.Client, ns, rcName string, replica } var _ = Describe("Nodes", func() { - supportedProviders := []string{"gce", "gke"} + supportedProviders := []string{"aws", "gce", "gke"} var testName string var c *client.Client var ns string